@llamaindex/llama-cloud 0.14.1

package/src/resources/pipelines/data-sources.ts

@@ -0,0 +1,245 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../core/resource';
+import * as DataSourcesAPI from '../data-sources';
+import * as Shared from '../shared';
+import * as PipelinesAPI from './pipelines';
+import { APIPromise } from '../../core/api-promise';
+import { RequestOptions } from '../../internal/request-options';
+import { path } from '../../internal/utils/path';
+
+export class DataSources extends APIResource {
+  /**
+   * Update the configuration of a data source in a pipeline.
+   */
+  update(
+    dataSourceID: string,
+    params: DataSourceUpdateParams,
+    options?: RequestOptions,
+  ): APIPromise<PipelineDataSource> {
+    const { pipeline_id, ...body } = params;
+    return this._client.put(path`/api/v1/pipelines/${pipeline_id}/data-sources/${dataSourceID}`, {
+      body,
+      ...options,
+    });
+  }
+
+  /**
+   * Get data sources for a pipeline.
+   */
+  getDataSources(pipelineID: string, options?: RequestOptions): APIPromise<DataSourceGetDataSourcesResponse> {
+    return this._client.get(path`/api/v1/pipelines/${pipelineID}/data-sources`, options);
+  }
+
+  /**
+   * Get the status of a data source for a pipeline.
+   */
+  getStatus(
+    dataSourceID: string,
+    params: DataSourceGetStatusParams,
+    options?: RequestOptions,
+  ): APIPromise<PipelinesAPI.ManagedIngestionStatusResponse> {
+    const { pipeline_id } = params;
+    return this._client.get(
+      path`/api/v1/pipelines/${pipeline_id}/data-sources/${dataSourceID}/status`,
+      options,
+    );
+  }
+
+  /**
+   * Run ingestion for the pipeline data source by incrementally updating the
+   * data-sink with upstream changes from data-source.
+   */
+  sync(
+    dataSourceID: string,
+    params: DataSourceSyncParams,
+    options?: RequestOptions,
+  ): APIPromise<PipelinesAPI.Pipeline> {
+    const { pipeline_id, ...body } = params;
+    return this._client.post(path`/api/v1/pipelines/${pipeline_id}/data-sources/${dataSourceID}/sync`, {
+      body,
+      ...options,
+    });
+  }
+
+  /**
+   * Add data sources to a pipeline.
+   */
+  updateDataSources(
+    pipelineID: string,
+    params: DataSourceUpdateDataSourcesParams,
+    options?: RequestOptions,
+  ): APIPromise<DataSourceUpdateDataSourcesResponse> {
+    const { body } = params;
+    return this._client.put(path`/api/v1/pipelines/${pipelineID}/data-sources`, { body: body, ...options });
+  }
+}
+
+/**
+ * Schema for a data source in a pipeline.
+ */
+export interface PipelineDataSource {
+  /**
+   * Unique identifier
+   */
+  id: string;
+
+  /**
+   * Component that implements the data source
+   */
+  component:
+    | { [key: string]: unknown }
+    | Shared.CloudS3DataSource
+    | Shared.CloudAzStorageBlobDataSource
+    | Shared.CloudOneDriveDataSource
+    | Shared.CloudSharepointDataSource
+    | Shared.CloudSlackDataSource
+    | Shared.CloudNotionPageDataSource
+    | Shared.CloudConfluenceDataSource
+    | Shared.CloudJiraDataSource
+    | Shared.CloudJiraDataSourceV2
+    | Shared.CloudBoxDataSource;
+
+  /**
+   * The ID of the data source.
+   */
+  data_source_id: string;
+
+  /**
+   * The last time the data source was automatically synced.
+   */
+  last_synced_at: string;
+
+  /**
+   * The name of the data source.
+   */
+  name: string;
+
+  /**
+   * The ID of the pipeline.
+   */
+  pipeline_id: string;
+
+  project_id: string;
+
+  source_type:
+    | 'S3'
+    | 'AZURE_STORAGE_BLOB'
+    | 'GOOGLE_DRIVE'
+    | 'MICROSOFT_ONEDRIVE'
+    | 'MICROSOFT_SHAREPOINT'
+    | 'SLACK'
+    | 'NOTION_PAGE'
+    | 'CONFLUENCE'
+    | 'JIRA'
+    | 'JIRA_V2'
+    | 'BOX';
+
+  /**
+   * Creation datetime
+   */
+  created_at?: string | null;
+
+  /**
+   * Custom metadata that will be present on all data loaded from the data source
+   */
+  custom_metadata?: {
+    [key: string]: { [key: string]: unknown } | Array<unknown> | string | number | boolean | null;
+  } | null;
+
+  /**
+   * The status of the data source in the pipeline.
+   */
+  status?: 'NOT_STARTED' | 'IN_PROGRESS' | 'SUCCESS' | 'ERROR' | 'CANCELLED' | null;
+
+  /**
+   * The last time the status was updated.
+   */
+  status_updated_at?: string | null;
+
+  /**
+   * The interval at which the data source should be synced.
+   */
+  sync_interval?: number | null;
+
+  /**
+   * The id of the user who set the sync schedule.
+   */
+  sync_schedule_set_by?: string | null;
+
+  /**
+   * Update datetime
+   */
+  updated_at?: string | null;
+
+  /**
+   * Version metadata for the data source
+   */
+  version_metadata?: DataSourcesAPI.DataSourceReaderVersionMetadata | null;
+}
+
+export type DataSourceGetDataSourcesResponse = Array<PipelineDataSource>;
+
+export type DataSourceUpdateDataSourcesResponse = Array<PipelineDataSource>;
+
+export interface DataSourceUpdateParams {
+  /**
+   * Path param
+   */
+  pipeline_id: string;
+
+  /**
+   * Body param: The interval at which the data source should be synced.
+   */
+  sync_interval?: number | null;
+}
+
+export interface DataSourceGetStatusParams {
+  pipeline_id: string;
+}
+
+export interface DataSourceSyncParams {
+  /**
+   * Path param
+   */
+  pipeline_id: string;
+
+  /**
+   * Body param
+   */
+  pipeline_file_ids?: Array<string> | null;
+}
+
+export interface DataSourceUpdateDataSourcesParams {
+  body: Array<DataSourceUpdateDataSourcesParams.Body>;
+}
+
+export namespace DataSourceUpdateDataSourcesParams {
+  /**
+   * Schema for creating an association between a data source and a pipeline.
+   */
+  export interface Body {
+    /**
+     * The ID of the data source.
+     */
+    data_source_id: string;
+
+    /**
+     * The interval at which the data source should be synced. Valid values are: 21600,
+     * 43200, 86400
+     */
+    sync_interval?: number | null;
+  }
+}
+
+export declare namespace DataSources {
+  export {
+    type PipelineDataSource as PipelineDataSource,
+    type DataSourceGetDataSourcesResponse as DataSourceGetDataSourcesResponse,
+    type DataSourceUpdateDataSourcesResponse as DataSourceUpdateDataSourcesResponse,
+    type DataSourceUpdateParams as DataSourceUpdateParams,
+    type DataSourceGetStatusParams as DataSourceGetStatusParams,
+    type DataSourceSyncParams as DataSourceSyncParams,
+    type DataSourceUpdateDataSourcesParams as DataSourceUpdateDataSourcesParams,
+  };
+}

package/src/resources/pipelines/documents.ts

@@ -0,0 +1,323 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../../core/resource';
+import * as PipelinesAPI from './pipelines';
+import { APIPromise } from '../../core/api-promise';
+import {
+  PagePromise,
+  PaginatedCloudDocuments,
+  type PaginatedCloudDocumentsParams,
+} from '../../core/pagination';
+import { buildHeaders } from '../../internal/headers';
+import { RequestOptions } from '../../internal/request-options';
+import { path } from '../../internal/utils/path';
+
+export class Documents extends APIResource {
+  /**
+   * Batch create documents for a pipeline.
+   */
+  create(
+    pipelineID: string,
+    params: DocumentCreateParams,
+    options?: RequestOptions,
+  ): APIPromise<DocumentCreateResponse> {
+    const { body } = params;
+    return this._client.post(path`/api/v1/pipelines/${pipelineID}/documents`, { body: body, ...options });
+  }
+
+  /**
+   * Return a list of documents for a pipeline.
+   */
+  list(
+    pipelineID: string,
+    query: DocumentListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<CloudDocumentsPaginatedCloudDocuments, CloudDocument> {
+    return this._client.getAPIList(
+      path`/api/v1/pipelines/${pipelineID}/documents/paginated`,
+      PaginatedCloudDocuments<CloudDocument>,
+      { query, ...options },
+    );
+  }
+
+  /**
+   * Delete a document from a pipeline. Initiates an async job that will:
+   *
+   * 1. Delete vectors from the vector store
+   * 2. Delete the document from MongoDB after vectors are successfully deleted
+   */
+  delete(documentID: string, params: DocumentDeleteParams, options?: RequestOptions): APIPromise<void> {
+    const { pipeline_id } = params;
+    return this._client.delete(path`/api/v1/pipelines/${pipeline_id}/documents/${documentID}`, {
+      ...options,
+      headers: buildHeaders([{ Accept: '*/*' }, options?.headers]),
+    });
+  }
+
+  /**
+   * Return a single document for a pipeline.
+   */
+  get(documentID: string, params: DocumentGetParams, options?: RequestOptions): APIPromise<CloudDocument> {
+    const { pipeline_id } = params;
+    return this._client.get(path`/api/v1/pipelines/${pipeline_id}/documents/${documentID}`, options);
+  }
+
+  /**
+   * Return a list of chunks for a pipeline document.
+   */
+  getChunks(
+    documentID: string,
+    params: DocumentGetChunksParams,
+    options?: RequestOptions,
+  ): APIPromise<DocumentGetChunksResponse> {
+    const { pipeline_id } = params;
+    return this._client.get(path`/api/v1/pipelines/${pipeline_id}/documents/${documentID}/chunks`, options);
+  }
+
+  /**
+   * Return a single document for a pipeline.
+   */
+  getStatus(
+    documentID: string,
+    params: DocumentGetStatusParams,
+    options?: RequestOptions,
+  ): APIPromise<PipelinesAPI.ManagedIngestionStatusResponse> {
+    const { pipeline_id } = params;
+    return this._client.get(path`/api/v1/pipelines/${pipeline_id}/documents/${documentID}/status`, options);
+  }
+
+  /**
+   * Sync a specific document for a pipeline.
+   */
+  sync(documentID: string, params: DocumentSyncParams, options?: RequestOptions): APIPromise<unknown> {
+    const { pipeline_id } = params;
+    return this._client.post(path`/api/v1/pipelines/${pipeline_id}/documents/${documentID}/sync`, options);
+  }
+
+  /**
+   * Batch create or update a document for a pipeline.
+   */
+  upsert(
+    pipelineID: string,
+    params: DocumentUpsertParams,
+    options?: RequestOptions,
+  ): APIPromise<DocumentUpsertResponse> {
+    const { body } = params;
+    return this._client.put(path`/api/v1/pipelines/${pipelineID}/documents`, { body: body, ...options });
+  }
+}
+
+export type CloudDocumentsPaginatedCloudDocuments = PaginatedCloudDocuments<CloudDocument>;
+
+/**
+ * Cloud document stored in S3.
+ */
+export interface CloudDocument {
+  id: string;
+
+  metadata: { [key: string]: unknown };
+
+  text: string;
+
+  excluded_embed_metadata_keys?: Array<string>;
+
+  excluded_llm_metadata_keys?: Array<string>;
+
+  /**
+   * indices in the CloudDocument.text where a new page begins. e.g. Second page
+   * starts at index specified by page_positions[1].
+   */
+  page_positions?: Array<number> | null;
+
+  status_metadata?: { [key: string]: unknown } | null;
+}
+
+/**
+ * Create a new cloud document.
+ */
+export interface CloudDocumentCreate {
+  metadata: { [key: string]: unknown };
+
+  text: string;
+
+  id?: string | null;
+
+  excluded_embed_metadata_keys?: Array<string>;
+
+  excluded_llm_metadata_keys?: Array<string>;
+
+  /**
+   * indices in the CloudDocument.text where a new page begins. e.g. Second page
+   * starts at index specified by page_positions[1].
+   */
+  page_positions?: Array<number> | null;
+}
+
+/**
+ * Provided for backward compatibility.
+ *
+ * Note: we keep the field with the typo "seperator" to maintain backward
+ * compatibility for serialized objects.
+ */
+export interface TextNode {
+  class_name?: string;
+
+  /**
+   * Embedding of the node.
+   */
+  embedding?: Array<number> | null;
+
+  /**
+   * End char index of the node.
+   */
+  end_char_idx?: number | null;
+
+  /**
+   * Metadata keys that are excluded from text for the embed model.
+   */
+  excluded_embed_metadata_keys?: Array<string>;
+
+  /**
+   * Metadata keys that are excluded from text for the LLM.
+   */
+  excluded_llm_metadata_keys?: Array<string>;
+
+  /**
+   * A flat dictionary of metadata fields
+   */
+  extra_info?: { [key: string]: unknown };
+
+  /**
+   * Unique ID of the node.
+   */
+  id_?: string;
+
+  /**
+   * Separator between metadata fields when converting to string.
+   */
+  metadata_seperator?: string;
+
+  /**
+   * Template for how metadata is formatted, with {key} and {value} placeholders.
+   */
+  metadata_template?: string;
+
+  /**
+   * MIME type of the node content.
+   */
+  mimetype?: string;
+
+  /**
+   * A mapping of relationships to other node information.
+   */
+  relationships?: { [key: string]: TextNode.RelatedNodeInfo | Array<TextNode.UnionMember1> };
+
+  /**
+   * Start char index of the node.
+   */
+  start_char_idx?: number | null;
+
+  /**
+   * Text content of the node.
+   */
+  text?: string;
+
+  /**
+   * Template for how text is formatted, with {content} and {metadata_str}
+   * placeholders.
+   */
+  text_template?: string;
+}
+
+export namespace TextNode {
+  export interface RelatedNodeInfo {
+    node_id: string;
+
+    class_name?: string;
+
+    hash?: string | null;
+
+    metadata?: { [key: string]: unknown };
+
+    node_type?: '1' | '2' | '3' | '4' | '5' | (string & {}) | null;
+  }
+
+  export interface UnionMember1 {
+    node_id: string;
+
+    class_name?: string;
+
+    hash?: string | null;
+
+    metadata?: { [key: string]: unknown };
+
+    node_type?: '1' | '2' | '3' | '4' | '5' | (string & {}) | null;
+  }
+}
+
+export type DocumentCreateResponse = Array<CloudDocument>;
+
+export type DocumentGetChunksResponse = Array<TextNode>;
+
+export type DocumentSyncResponse = unknown;
+
+export type DocumentUpsertResponse = Array<CloudDocument>;
+
+export interface DocumentCreateParams {
+  body: Array<CloudDocumentCreate>;
+}
+
+export interface DocumentListParams extends PaginatedCloudDocumentsParams {
+  file_id?: string | null;
+
+  only_api_data_source_documents?: boolean | null;
+
+  only_direct_upload?: boolean | null;
+
+  status_refresh_policy?: 'cached' | 'ttl';
+}
+
+export interface DocumentDeleteParams {
+  pipeline_id: string;
+}
+
+export interface DocumentGetParams {
+  pipeline_id: string;
+}
+
+export interface DocumentGetChunksParams {
+  pipeline_id: string;
+}
+
+export interface DocumentGetStatusParams {
+  pipeline_id: string;
+}
+
+export interface DocumentSyncParams {
+  pipeline_id: string;
+}
+
+export interface DocumentUpsertParams {
+  body: Array<CloudDocumentCreate>;
+}
+
+export declare namespace Documents {
+  export {
+    type CloudDocument as CloudDocument,
+    type CloudDocumentCreate as CloudDocumentCreate,
+    type TextNode as TextNode,
+    type DocumentCreateResponse as DocumentCreateResponse,
+    type DocumentGetChunksResponse as DocumentGetChunksResponse,
+    type DocumentSyncResponse as DocumentSyncResponse,
+    type DocumentUpsertResponse as DocumentUpsertResponse,
+    type CloudDocumentsPaginatedCloudDocuments as CloudDocumentsPaginatedCloudDocuments,
+    type DocumentCreateParams as DocumentCreateParams,
+    type DocumentListParams as DocumentListParams,
+    type DocumentDeleteParams as DocumentDeleteParams,
+    type DocumentGetParams as DocumentGetParams,
+    type DocumentGetChunksParams as DocumentGetChunksParams,
+    type DocumentGetStatusParams as DocumentGetStatusParams,
+    type DocumentSyncParams as DocumentSyncParams,
+    type DocumentUpsertParams as DocumentUpsertParams,
+  };
+}