@whaly/connector-sdk 0.3.7 → 0.3.8

package/dist/index.d.mts

@@ -1,4 +1,4 @@
-import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
+import { AxiosRequestHeaders, AxiosInstance, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
 import Ajv, { ValidateFunction, ErrorObject } from 'ajv';
 import { Dayjs } from 'dayjs';
 import { WriteStream } from 'fs';
@@ -266,6 +266,7 @@ interface BaseConfig {
     schema: string;
     syncedAtColumnName?: string;
     syncedAtColumnUseLegacyStringType?: boolean;
+    outputDir?: string;
 }
 
 interface ColumnMappingStoreUnsafeToSafe {
@@ -349,7 +350,8 @@ declare class StreamState$1 {
     _hasBeenLoadedYetDuringThisSync: boolean;
     replicationMethod: ReplicationMethod;
     compiledValidateFn?: ValidateFunction;
-    constructor(streamId: StreamId, dbSync: StreamWarehouseSyncService, replicationMethod: ReplicationMethod);
+    tmpDir: string;
+    constructor(streamId: StreamId, dbSync: StreamWarehouseSyncService, replicationMethod: ReplicationMethod, tmpDir?: string);
     setSchema(schema: FlattenedSchema): void;
     getSchema(): FlattenedSchema | undefined;
     setCompiledValidateFn(fn: ValidateFunction): void;
@@ -382,6 +384,8 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     schemaHooks: TargetSchemaHook[];
     syncTime: Dayjs;
     stateProvider: StateProvider;
+    readonly outputDir: string;
+    readonly tmpDir: string;
     batchedState: any;
     flushedState: any;
     renameColumnStore: RenameColumnStore;
@@ -408,6 +412,7 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     private loadAllStreamsInWarehouse;
     private emitState;
     private uploadStreamsToWarehouse;
+    private logSyncSummary;
     static uploadSingleStreamToWarehouse: (streamState: StreamState$1) => Promise<void>;
 }
 
@@ -693,7 +698,7 @@ declare const addWhalyFields: (record: any, batchDate: string, columnName?: stri
 
 declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSchema;
 
-declare const createTemporaryFileStream: (streamId: StreamId) => TempFile;
+declare const createTemporaryFileStream: (streamId: StreamId, tmpDir?: string) => TempFile;
 declare function safePath(path: string): string;
 
 /**
@@ -1356,6 +1361,199 @@ declare class ImageTransform {
     static toWebp(inputPath: string, options: WebpTransformOptions): Promise<string>;
 }
 
+/**
+ * Metadata-only representation of a document in the source system.
+ * Yielded by DocumentStream.listDocuments().
+ */
+interface DocumentEntry {
+    /** Unique ID in source system — the reconciliation key. */
+    externalId: string;
+    /** Display name in Whaly. */
+    fileName: string;
+    /** Original filename in source system. */
+    originalFileName: string;
+    /** Original file path in source system. */
+    originalFilePath?: string;
+    /** Document author name. */
+    originalAuthor?: string;
+    /** File extension without dot (e.g. "pdf", "xlsx"). */
+    extension: string;
+    /** Source last-modified timestamp. Used for incremental skip. */
+    lastModified?: Date;
+    /** ISO date string: when the document becomes valid. */
+    validFrom?: string;
+    /** ISO date string: when the document expires. */
+    validUntil?: string;
+    /** Custom key-value metadata. */
+    metadata?: Record<string, string>;
+}
+/**
+ * A document as it exists in Whaly (returned by the Document API).
+ */
+interface WhalyDocument {
+    id: string;
+    file_name: string;
+    external_id: string;
+    original_file_name: string;
+    original_file_path: string;
+    original_author: string;
+    extension: string;
+    file_path: string;
+    valid_from: string;
+    valid_until: string;
+    size_kb: number;
+    storage: string;
+    metadata: Record<string, string> | null;
+}
+/** Paginated response from the Whaly Document API. */
+interface WhalyPaginatedResponse<T> {
+    data: T[];
+    paging: {
+        next?: {
+            after: string;
+        };
+    };
+}
+type DocumentStatus = "created" | "updated" | "reuploaded" | "deleted" | "skipped" | "error";
+interface DocumentManifestEntry {
+    externalId: string;
+    fileName: string;
+    status: DocumentStatus;
+    error?: string;
+}
+interface DocumentStreamManifest {
+    streamId: string;
+    syncedAt: string;
+    documents: DocumentManifestEntry[];
+    summary: DocumentSummary;
+}
+interface DocumentSummary {
+    total: number;
+    created: number;
+    updated: number;
+    reuploaded: number;
+    deleted: number;
+    skipped: number;
+    errors: number;
+}
+interface DocumentManifest {
+    syncedAt: string;
+    streams: DocumentStreamManifest[];
+    summary: DocumentSummary;
+}
+declare function emptyDocumentSummary(): DocumentSummary;
+declare function addDocumentSummaries(a: DocumentSummary, b: DocumentSummary): DocumentSummary;
+
+declare class DocumentDownloadSkipError extends Error {
+    constructor(message: string);
+}
+
+declare abstract class DocumentStream<C> {
+    readonly streamId: string;
+    readonly config: C;
+    constructor(streamId: string, config: C);
+    /** Yield all documents from the source (metadata only, no file download). */
+    abstract listDocuments(): AsyncIterable<DocumentEntry>;
+    /** Download a single document file to `destPath` on local disk. */
+    abstract downloadDocument(entry: DocumentEntry, destPath: string): Promise<void>;
+    /**
+     * Escape hatch: should the file be re-uploaded?
+     * Default: true (conservative — always re-upload).
+     * Override to compare timestamps or hashes for incremental skip.
+     */
+    shouldReupload(_sourceEntry: DocumentEntry, _existingDoc: WhalyDocument): boolean;
+    /**
+     * Escape hatch: does the metadata need updating?
+     * Default: compares fileName, validFrom, validUntil, originalAuthor, metadata.
+     */
+    shouldUpdateMetadata(sourceEntry: DocumentEntry, existingDoc: WhalyDocument): boolean;
+    /**
+     * Escape hatch: should this orphaned document be deleted?
+     * Default: true (delete all documents no longer in source).
+     */
+    shouldDelete(_orphanedDoc: WhalyDocument): boolean;
+}
+
+interface WhalyDocumentServiceConfig {
+    /** e.g. "https://org.my.whaly.io" */
+    apiEndpoint?: string;
+    /** e.g. "sk:xxxx" */
+    serviceAccountKey?: string;
+    /** Object storage ID for file uploads. */
+    objectStorageId: string;
+}
+interface WhalyUploadResult {
+    storage: string;
+    filePath: string;
+    sizeKb: number;
+}
+declare class WhalyDocumentService {
+    private axiosClient;
+    readonly objectStorageId: string;
+    private gcsBucket;
+    constructor(config: WhalyDocumentServiceConfig);
+    /** Fetch all documents from the API, handling cursor-based pagination. */
+    listAllDocuments(): Promise<WhalyDocument[]>;
+    /** Upload a file to object storage. Returns storage path and size. */
+    uploadFile(destinationPath: string, localFilePath: string, fileName: string): Promise<WhalyUploadResult>;
+    private uploadFileViaGCS;
+    private uploadFileViaAPI;
+    /** Create a new document record. */
+    createDocument(payload: Omit<WhalyDocument, "id">): Promise<WhalyDocument>;
+    /** Update an existing document record. */
+    updateDocument(id: string, payload: Partial<WhalyDocument>): Promise<WhalyDocument>;
+    /** Delete a document record. */
+    deleteDocument(id: string): Promise<void>;
+}
+
+declare class WhalyDocumentTarget {
+    readonly config: WhalyDocumentServiceConfig;
+    private service;
+    constructor(config: WhalyDocumentServiceConfig);
+    /** Fetch all existing documents from Whaly. */
+    listExistingDocuments(): Promise<WhalyDocument[]>;
+    /**
+     * Create a new document: upload file to object storage, then create the document record.
+     * @param streamId Used to namespace the file path in object storage.
+     */
+    createDocument(streamId: string, entry: DocumentEntry, localFilePath: string): Promise<void>;
+    /**
+     * Re-upload the file and update the document record.
+     */
+    reuploadDocument(streamId: string, docId: string, entry: DocumentEntry, localFilePath: string): Promise<void>;
+    /** Update only the metadata fields (no file re-upload). */
+    updateDocumentMetadata(docId: string, entry: DocumentEntry, existingDoc: WhalyDocument): Promise<void>;
+    /** Delete a document record. */
+    deleteDocument(docId: string, externalId: string): Promise<void>;
+}
+
+declare abstract class DocumentTap<C> {
+    readonly config: C;
+    readonly outputDir: string;
+    readonly concurrency: number;
+    /**
+     * The single stream for this tap.
+     * Set during init(). A DocumentTap supports exactly one stream to avoid
+     * cross-stream deletion issues (existing docs cannot be filtered by stream).
+     */
+    stream: DocumentStream<unknown>;
+    target: WhalyDocumentTarget;
+    constructor(config: C, outputDir?: string, concurrency?: number);
+    /**
+     * Initialize the tap: set `this.stream` to the single DocumentStream.
+     * Called once at the start of sync().
+     */
+    abstract init(): Promise<void>;
+    sync(): Promise<DocumentManifest>;
+    private collectSourceEntries;
+    private computeDiff;
+    private executeCreate;
+    private executeReupload;
+    private executeMetadataUpdate;
+    private executeDelete;
+    private computeSummary;
+}
+
 interface BigQueryConfig extends BaseConfig {
     schema: string;
     project_id: string;
@@ -1437,4 +1635,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WriteGeneratorToCsvOptions, _greaterThan, addWhalyFields, checkCsvHeaderRow, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
+import { AxiosRequestHeaders, AxiosInstance, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
 import Ajv, { ValidateFunction, ErrorObject } from 'ajv';
 import { Dayjs } from 'dayjs';
 import { WriteStream } from 'fs';
@@ -266,6 +266,7 @@ interface BaseConfig {
     schema: string;
     syncedAtColumnName?: string;
     syncedAtColumnUseLegacyStringType?: boolean;
+    outputDir?: string;
 }
 
 interface ColumnMappingStoreUnsafeToSafe {
@@ -349,7 +350,8 @@ declare class StreamState$1 {
     _hasBeenLoadedYetDuringThisSync: boolean;
     replicationMethod: ReplicationMethod;
     compiledValidateFn?: ValidateFunction;
-    constructor(streamId: StreamId, dbSync: StreamWarehouseSyncService, replicationMethod: ReplicationMethod);
+    tmpDir: string;
+    constructor(streamId: StreamId, dbSync: StreamWarehouseSyncService, replicationMethod: ReplicationMethod, tmpDir?: string);
     setSchema(schema: FlattenedSchema): void;
     getSchema(): FlattenedSchema | undefined;
     setCompiledValidateFn(fn: ValidateFunction): void;
@@ -382,6 +384,8 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     schemaHooks: TargetSchemaHook[];
     syncTime: Dayjs;
     stateProvider: StateProvider;
+    readonly outputDir: string;
+    readonly tmpDir: string;
     batchedState: any;
     flushedState: any;
     renameColumnStore: RenameColumnStore;
@@ -408,6 +412,7 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     private loadAllStreamsInWarehouse;
     private emitState;
     private uploadStreamsToWarehouse;
+    private logSyncSummary;
     static uploadSingleStreamToWarehouse: (streamState: StreamState$1) => Promise<void>;
 }
 
@@ -693,7 +698,7 @@ declare const addWhalyFields: (record: any, batchDate: string, columnName?: stri
 
 declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSchema;
 
-declare const createTemporaryFileStream: (streamId: StreamId) => TempFile;
+declare const createTemporaryFileStream: (streamId: StreamId, tmpDir?: string) => TempFile;
 declare function safePath(path: string): string;
 
 /**
@@ -1356,6 +1361,199 @@ declare class ImageTransform {
     static toWebp(inputPath: string, options: WebpTransformOptions): Promise<string>;
 }
 
+/**
+ * Metadata-only representation of a document in the source system.
+ * Yielded by DocumentStream.listDocuments().
+ */
+interface DocumentEntry {
+    /** Unique ID in source system — the reconciliation key. */
+    externalId: string;
+    /** Display name in Whaly. */
+    fileName: string;
+    /** Original filename in source system. */
+    originalFileName: string;
+    /** Original file path in source system. */
+    originalFilePath?: string;
+    /** Document author name. */
+    originalAuthor?: string;
+    /** File extension without dot (e.g. "pdf", "xlsx"). */
+    extension: string;
+    /** Source last-modified timestamp. Used for incremental skip. */
+    lastModified?: Date;
+    /** ISO date string: when the document becomes valid. */
+    validFrom?: string;
+    /** ISO date string: when the document expires. */
+    validUntil?: string;
+    /** Custom key-value metadata. */
+    metadata?: Record<string, string>;
+}
+/**
+ * A document as it exists in Whaly (returned by the Document API).
+ */
+interface WhalyDocument {
+    id: string;
+    file_name: string;
+    external_id: string;
+    original_file_name: string;
+    original_file_path: string;
+    original_author: string;
+    extension: string;
+    file_path: string;
+    valid_from: string;
+    valid_until: string;
+    size_kb: number;
+    storage: string;
+    metadata: Record<string, string> | null;
+}
+/** Paginated response from the Whaly Document API. */
+interface WhalyPaginatedResponse<T> {
+    data: T[];
+    paging: {
+        next?: {
+            after: string;
+        };
+    };
+}
+type DocumentStatus = "created" | "updated" | "reuploaded" | "deleted" | "skipped" | "error";
+interface DocumentManifestEntry {
+    externalId: string;
+    fileName: string;
+    status: DocumentStatus;
+    error?: string;
+}
+interface DocumentStreamManifest {
+    streamId: string;
+    syncedAt: string;
+    documents: DocumentManifestEntry[];
+    summary: DocumentSummary;
+}
+interface DocumentSummary {
+    total: number;
+    created: number;
+    updated: number;
+    reuploaded: number;
+    deleted: number;
+    skipped: number;
+    errors: number;
+}
+interface DocumentManifest {
+    syncedAt: string;
+    streams: DocumentStreamManifest[];
+    summary: DocumentSummary;
+}
+declare function emptyDocumentSummary(): DocumentSummary;
+declare function addDocumentSummaries(a: DocumentSummary, b: DocumentSummary): DocumentSummary;
+
+declare class DocumentDownloadSkipError extends Error {
+    constructor(message: string);
+}
+
+declare abstract class DocumentStream<C> {
+    readonly streamId: string;
+    readonly config: C;
+    constructor(streamId: string, config: C);
+    /** Yield all documents from the source (metadata only, no file download). */
+    abstract listDocuments(): AsyncIterable<DocumentEntry>;
+    /** Download a single document file to `destPath` on local disk. */
+    abstract downloadDocument(entry: DocumentEntry, destPath: string): Promise<void>;
+    /**
+     * Escape hatch: should the file be re-uploaded?
+     * Default: true (conservative — always re-upload).
+     * Override to compare timestamps or hashes for incremental skip.
+     */
+    shouldReupload(_sourceEntry: DocumentEntry, _existingDoc: WhalyDocument): boolean;
+    /**
+     * Escape hatch: does the metadata need updating?
+     * Default: compares fileName, validFrom, validUntil, originalAuthor, metadata.
+     */
+    shouldUpdateMetadata(sourceEntry: DocumentEntry, existingDoc: WhalyDocument): boolean;
+    /**
+     * Escape hatch: should this orphaned document be deleted?
+     * Default: true (delete all documents no longer in source).
+     */
+    shouldDelete(_orphanedDoc: WhalyDocument): boolean;
+}
+
+interface WhalyDocumentServiceConfig {
+    /** e.g. "https://org.my.whaly.io" */
+    apiEndpoint?: string;
+    /** e.g. "sk:xxxx" */
+    serviceAccountKey?: string;
+    /** Object storage ID for file uploads. */
+    objectStorageId: string;
+}
+interface WhalyUploadResult {
+    storage: string;
+    filePath: string;
+    sizeKb: number;
+}
+declare class WhalyDocumentService {
+    private axiosClient;
+    readonly objectStorageId: string;
+    private gcsBucket;
+    constructor(config: WhalyDocumentServiceConfig);
+    /** Fetch all documents from the API, handling cursor-based pagination. */
+    listAllDocuments(): Promise<WhalyDocument[]>;
+    /** Upload a file to object storage. Returns storage path and size. */
+    uploadFile(destinationPath: string, localFilePath: string, fileName: string): Promise<WhalyUploadResult>;
+    private uploadFileViaGCS;
+    private uploadFileViaAPI;
+    /** Create a new document record. */
+    createDocument(payload: Omit<WhalyDocument, "id">): Promise<WhalyDocument>;
+    /** Update an existing document record. */
+    updateDocument(id: string, payload: Partial<WhalyDocument>): Promise<WhalyDocument>;
+    /** Delete a document record. */
+    deleteDocument(id: string): Promise<void>;
+}
+
+declare class WhalyDocumentTarget {
+    readonly config: WhalyDocumentServiceConfig;
+    private service;
+    constructor(config: WhalyDocumentServiceConfig);
+    /** Fetch all existing documents from Whaly. */
+    listExistingDocuments(): Promise<WhalyDocument[]>;
+    /**
+     * Create a new document: upload file to object storage, then create the document record.
+     * @param streamId Used to namespace the file path in object storage.
+     */
+    createDocument(streamId: string, entry: DocumentEntry, localFilePath: string): Promise<void>;
+    /**
+     * Re-upload the file and update the document record.
+     */
+    reuploadDocument(streamId: string, docId: string, entry: DocumentEntry, localFilePath: string): Promise<void>;
+    /** Update only the metadata fields (no file re-upload). */
+    updateDocumentMetadata(docId: string, entry: DocumentEntry, existingDoc: WhalyDocument): Promise<void>;
+    /** Delete a document record. */
+    deleteDocument(docId: string, externalId: string): Promise<void>;
+}
+
+declare abstract class DocumentTap<C> {
+    readonly config: C;
+    readonly outputDir: string;
+    readonly concurrency: number;
+    /**
+     * The single stream for this tap.
+     * Set during init(). A DocumentTap supports exactly one stream to avoid
+     * cross-stream deletion issues (existing docs cannot be filtered by stream).
+     */
+    stream: DocumentStream<unknown>;
+    target: WhalyDocumentTarget;
+    constructor(config: C, outputDir?: string, concurrency?: number);
+    /**
+     * Initialize the tap: set `this.stream` to the single DocumentStream.
+     * Called once at the start of sync().
+     */
+    abstract init(): Promise<void>;
+    sync(): Promise<DocumentManifest>;
+    private collectSourceEntries;
+    private computeDiff;
+    private executeCreate;
+    private executeReupload;
+    private executeMetadataUpdate;
+    private executeDelete;
+    private computeSummary;
+}
+
 interface BigQueryConfig extends BaseConfig {
     schema: string;
     project_id: string;
@@ -1437,4 +1635,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WriteGeneratorToCsvOptions, _greaterThan, addWhalyFields, checkCsvHeaderRow, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, type AssetEntry, type AssetManifest, type AssetManifestEntry, type AssetReplicationMode, AssetStream, AssetTap, AssetTarget, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CdnAssetTarget, type CdnAssetTargetConfig, type CdnFileMetadata, CdnService, type CdnServiceConfig, type CdnUploadResult, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, DocumentDownloadSkipError, type DocumentEntry, type DocumentManifest, type DocumentManifestEntry, type DocumentStatus, DocumentStream, type DocumentStreamManifest, type DocumentSummary, DocumentTap, type DownloadFilesOptions, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, ImageTransform, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type ListFilesOptions, MIME_TYPES, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProcessedAsset, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamManifest, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, type WebpTransformOptions, type WhalyDocument, WhalyDocumentService, type WhalyDocumentServiceConfig, WhalyDocumentTarget, type WhalyPaginatedResponse, type WhalyUploadResult, type WriteGeneratorToCsvOptions, _greaterThan, addDocumentSummaries, addWhalyFields, checkCsvHeaderRow, collectPaginated, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, downloadFiles, emptyDocumentSummary, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getApiEndpoint, getAxiosInstance, getCdnId, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getDryRunLimit, getJSONApiCall, getJSONApiCallWithFullResponse, getMimeType, getRawJSONApiCall, getServiceAccountKey, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, isDryRun, listFilesRecursively, loadJson, loadSchema, logger, optionalEnv, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, processFromAsyncIterable, removeParasiteProperties, requireEnv, rowGeneratorFromCsv, runWithConcurrency, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };