@whaly/connector-sdk 0.3.0 → 0.3.1

package/dist/index.d.mts

@@ -1,5 +1,5 @@
 import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
-import { Validator, ValidationError } from 'jsonschema';
+import Ajv, { ValidateFunction, ErrorObject } from 'ajv';
 import { Dayjs } from 'dayjs';
 import { WriteStream } from 'fs';
 import { WorkBook } from 'xlsx';
@@ -273,9 +273,12 @@ declare class StreamState$1 {
     keyProperties: string[];
     _hasBeenLoadedYetDuringThisSync: boolean;
     replicationMethod: ReplicationMethod;
+    compiledValidateFn?: ValidateFunction;
     constructor(streamId: StreamId, dbSync: StreamWarehouseSyncService, replicationMethod: ReplicationMethod);
     setSchema(schema: FlattenedSchema): void;
     getSchema(): FlattenedSchema | undefined;
+    setCompiledValidateFn(fn: ValidateFunction): void;
+    getCompiledValidateFn(): ValidateFunction | undefined;
     getBatchDate(): string;
     getBatchedRowCount(): number;
     incrementBatchedRowCount(): void;
@@ -308,7 +311,7 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     streams: {
         [streamName: string]: StreamState$1;
     };
-    validator: Validator;
+    ajv: Ajv;
     constructor(config: C, stateProvider: StateProvider);
     static requiredConfigKeys: string[];
     abstract newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => StreamWarehouseSyncService;
@@ -414,6 +417,7 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
     isSorted: boolean;
     isSilent: boolean;
     childConcurrency: number;
+    totalRows?: number;
     rowsSyncedMetricsConf: MetricConfiguration;
     executionTimeMetricsConf: MetricConfiguration;
     constructor(config: C, tapState: InputTapState, target: ITarget, childConcurrency?: number);
@@ -595,8 +599,8 @@ declare abstract class Tap<C> {
 
 declare class SchemaValidationError extends Error {
     streamId: StreamId;
-    validationErrors: ValidationError[];
-    constructor(message: string, streamId: StreamId, errors: ValidationError[]);
+    validationErrors: ErrorObject[];
+    constructor(message: string, streamId: StreamId, errors: ErrorObject[]);
 }
 declare class MissingFieldInSchemaError extends Error {
     streamId: StreamId;
@@ -808,6 +812,11 @@ declare class ExcelExtractionConfigBuilder {
  * Supports custom separators, encoding, BOM stripping, and field transformers.
  */
 declare function rowGeneratorFromCsv(path: string, fileConfig: CsvFileConfig): AsyncGenerator<Record<string, any>>;
+/**
+ * Counts data rows in a CSV file by scanning for newline bytes.
+ * Subtracts 1 for the header row. Fast: no CSV parsing required.
+ */
+declare function countCsvLines(filePath: string): Promise<number>;
 /**
  * Validates CSV header row against expected configuration.
  */
@@ -885,6 +894,7 @@ declare class FileStream extends Stream<Record<string, any>, FileStreamConfig> {
     /**
      * Yield records from all file(s).
      * When multiple file paths are provided, records are yielded sequentially from each file.
+     * Pre-counts total rows across all files to enable percentage progress logging.
      */
     _getRecords(): AsyncIterable<Record<string, any>>;
 }
@@ -1047,6 +1057,8 @@ declare class CloudStorageService {
     writeStringObject(objectPath: string, contents: string): Promise<void>;
     /**
      * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
+     * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
+     * or `<prefix>/default/` otherwise.
      * Returns the GCS File reference.
      */
     uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
@@ -1131,4 +1143,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { AxiosInstance, AxiosRequestHeaders, AxiosResponse, AxiosRequestConfig, AxiosError } from 'axios';
-import { Validator, ValidationError } from 'jsonschema';
+import Ajv, { ValidateFunction, ErrorObject } from 'ajv';
 import { Dayjs } from 'dayjs';
 import { WriteStream } from 'fs';
 import { WorkBook } from 'xlsx';
@@ -273,9 +273,12 @@ declare class StreamState$1 {
     keyProperties: string[];
     _hasBeenLoadedYetDuringThisSync: boolean;
     replicationMethod: ReplicationMethod;
+    compiledValidateFn?: ValidateFunction;
     constructor(streamId: StreamId, dbSync: StreamWarehouseSyncService, replicationMethod: ReplicationMethod);
     setSchema(schema: FlattenedSchema): void;
     getSchema(): FlattenedSchema | undefined;
+    setCompiledValidateFn(fn: ValidateFunction): void;
+    getCompiledValidateFn(): ValidateFunction | undefined;
     getBatchDate(): string;
     getBatchedRowCount(): number;
     incrementBatchedRowCount(): void;
@@ -308,7 +311,7 @@ declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     streams: {
         [streamName: string]: StreamState$1;
     };
-    validator: Validator;
+    ajv: Ajv;
     constructor(config: C, stateProvider: StateProvider);
     static requiredConfigKeys: string[];
     abstract newDBSyncInstance: (streamId: StreamId, database: string, schema: string, table: string) => StreamWarehouseSyncService;
@@ -414,6 +417,7 @@ declare abstract class Stream<O = any, C = any, P = undefined> {
     isSorted: boolean;
     isSilent: boolean;
     childConcurrency: number;
+    totalRows?: number;
     rowsSyncedMetricsConf: MetricConfiguration;
     executionTimeMetricsConf: MetricConfiguration;
     constructor(config: C, tapState: InputTapState, target: ITarget, childConcurrency?: number);
@@ -595,8 +599,8 @@ declare abstract class Tap<C> {
 
 declare class SchemaValidationError extends Error {
     streamId: StreamId;
-    validationErrors: ValidationError[];
-    constructor(message: string, streamId: StreamId, errors: ValidationError[]);
+    validationErrors: ErrorObject[];
+    constructor(message: string, streamId: StreamId, errors: ErrorObject[]);
 }
 declare class MissingFieldInSchemaError extends Error {
     streamId: StreamId;
@@ -808,6 +812,11 @@ declare class ExcelExtractionConfigBuilder {
  * Supports custom separators, encoding, BOM stripping, and field transformers.
  */
 declare function rowGeneratorFromCsv(path: string, fileConfig: CsvFileConfig): AsyncGenerator<Record<string, any>>;
+/**
+ * Counts data rows in a CSV file by scanning for newline bytes.
+ * Subtracts 1 for the header row. Fast: no CSV parsing required.
+ */
+declare function countCsvLines(filePath: string): Promise<number>;
 /**
  * Validates CSV header row against expected configuration.
  */
@@ -885,6 +894,7 @@ declare class FileStream extends Stream<Record<string, any>, FileStreamConfig> {
     /**
      * Yield records from all file(s).
      * When multiple file paths are provided, records are yielded sequentially from each file.
+     * Pre-counts total rows across all files to enable percentage progress logging.
      */
     _getRecords(): AsyncIterable<Record<string, any>>;
 }
@@ -1047,6 +1057,8 @@ declare class CloudStorageService {
     writeStringObject(objectPath: string, contents: string): Promise<void>;
     /**
      * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
+     * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
+     * or `<prefix>/default/` otherwise.
      * Returns the GCS File reference.
      */
     uploadFileWithUniqueName(filePath: string, prefix: string, streamId: string): Promise<File>;
@@ -1131,4 +1143,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };

package/dist/index.js

@@ -68,6 +68,7 @@ __export(index_exports, {
   _greaterThan: () => _greaterThan,
   addWhalyFields: () => addWhalyFields,
   checkCsvHeaderRow: () => checkCsvHeaderRow,
+  countCsvLines: () => countCsvLines,
   createCellReference: () => createCellReference,
   createCsvStreamConfig: () => createCsvStreamConfig,
   createExcelGenerator: () => createExcelGenerator,
@@ -847,6 +848,8 @@ var Stream = class {
   // However, a STATE will still be emitted if the Stream is using an incremental sync type.
   isSilent = false;
   childConcurrency;
+  // Optional total row count for percentage progress logging
+  totalRows;
   // Metrics configuration
   rowsSyncedMetricsConf;
   executionTimeMetricsConf;
@@ -1124,6 +1127,10 @@ var Stream = class {
         metric.increment();
       });
       rowsSent += 1;
+      if (rowsSent % 1e3 === 0) {
+        const percentStr = this.totalRows ? ` (${Math.round(rowsSent / this.totalRows * 100)}%)` : "";
+        logger.info(`\u23F3 Stream: ${this.streamId} - ${rowsSent} records synced so far...${percentStr}`);
+      }
     }
     const stateServiceInst = StateService.getInstance();
     const state = stateServiceInst.getBookmark(this.streamId);
@@ -1796,7 +1803,8 @@ var flattenSchema = (streamId, schema) => {
 };
 
 // src/sdk/models/target/target.ts
-var import_jsonschema = require("jsonschema");
+var import_ajv = __toESM(require("ajv"));
+var import_ajv_formats = __toESM(require("ajv-formats"));
 var import_dayjs4 = __toESM(require("dayjs"));
 
 // src/sdk/models/target/streamDbState.ts
@@ -1874,6 +1882,8 @@ var StreamState = class {
   // So we need to keep track of whether we are at the first load or not
   _hasBeenLoadedYetDuringThisSync;
   replicationMethod;
+  // Pre-compiled ajv ValidateFunction — compiled once per schema, reused for every row
+  compiledValidateFn;
   constructor(streamId, dbSync, replicationMethod) {
     this.streamId = streamId;
     this.dbSync = dbSync;
@@ -1891,6 +1901,12 @@ var StreamState = class {
   getSchema() {
     return this.schema;
   }
+  setCompiledValidateFn(fn) {
+    this.compiledValidateFn = fn;
+  }
+  getCompiledValidateFn() {
+    return this.compiledValidateFn;
+  }
   getBatchDate() {
     return this.batchDate.format(defaultDateTimeFormat);
   }
@@ -1947,8 +1963,8 @@ var ITarget = class _ITarget {
   flushedState;
   renameColumnStore;
   streams;
-  // JSON Schema validator
-  validator;
+  // JSON Schema validator (ajv — pre-compiles schemas into optimized JS functions)
+  ajv;
   constructor(config, stateProvider) {
     this.config = config;
     this.stateProvider = stateProvider;
@@ -1956,7 +1972,7 @@ var ITarget = class _ITarget {
     this.streams = {};
     this.batchedState = {};
     this.flushedState = {};
-    this.validator = new import_jsonschema.Validator();
+    this.ajv = (0, import_ajv_formats.default)(new import_ajv.default({ allErrors: true, strict: false }));
     this.syncTime = (0, import_dayjs4.default)();
     this.renameColumnStore = new RenameColumnStore();
   }
@@ -2015,12 +2031,14 @@ var ITarget = class _ITarget {
       if (!stream) {
         throw new MissingSchemaError(`A record for stream:\`${streamId}\` was encountered before a corresponding schema.`, streamId);
       }
-      await semaphore.runExclusive(async () => {
-        const shouldUploadIntermediateBatch = this.shouldUploadIntermediateBatch(streamId);
-        if (shouldUploadIntermediateBatch) {
-          await this.loadAllStreamsInWarehouse({ isFinalLoad: false });
-        }
-      });
+      if (stream.getBatchedRowCount() % 1e4 === 0) {
+        await semaphore.runExclusive(async () => {
+          const shouldUploadIntermediateBatch = this.shouldUploadIntermediateBatch(streamId);
+          if (shouldUploadIntermediateBatch) {
+            await this.loadAllStreamsInWarehouse({ isFinalLoad: false });
+          }
+        });
+      }
       const record = message.record;
       const schema = stream.getSchema();
       if (!schema) {
@@ -2029,13 +2047,13 @@ var ITarget = class _ITarget {
       }
       const recordWithoutParasiteProperties = removeParasiteProperties(record, schema);
       const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate());
-      const validationResult = this.validator.validate(recordWithWhalyFields, schema);
-      if (!validationResult.valid) {
+      const validateFn = stream.getCompiledValidateFn();
+      if (validateFn && !validateFn(recordWithWhalyFields)) {
         throw new SchemaValidationError(`Stream: ${streamId} - Record is not valid according to schema.
-                Validation errors: ${JSON.stringify(validationResult.errors)}
+                Validation errors: ${JSON.stringify(validateFn.errors)}
 
                 Record: ${JSON.stringify(recordWithoutParasiteProperties)}
-                Schema: ${JSON.stringify(stream.getSchema())}`, streamId, validationResult.errors);
+                Schema: ${JSON.stringify(stream.getSchema())}`, streamId, validateFn.errors ?? []);
       }
       const recordWithValidDate = this.validateDateRange(recordWithWhalyFields, schema);
       const recordWithDecimal = this.convertNumberIntoDecimal(recordWithValidDate, schema);
@@ -2124,6 +2142,7 @@ var ITarget = class _ITarget {
         }
       });
       streamState.setSchema(schema);
+      streamState.setCompiledValidateFn(this.ajv.compile({ type: "object", properties: schema }));
       const streamReplicationMethod = this.streams[streamId]?.getReplicationMethod();
       if (message.keyProperties.length === 0 && streamReplicationMethod !== "APPEND" /* APPEND */) {
         throw new Error(`StreamId: ${message.stream} - \`key_properties\` field is required and can't be empty in SCHEMA message.
@@ -2609,6 +2628,7 @@ Actual:   "${actual}" (${actualInfo.length} chars) [${actualInfo.hex}]`;
 };
 async function* rowGeneratorFromCsv(path2, fileConfig) {
   const isGeneratorConfigArray = Array.isArray(fileConfig.fields);
+  const configKeys = isGeneratorConfigArray ? [] : Object.keys(fileConfig.fields);
   const csvOptions = { separator: fileConfig.separator };
   if (isGeneratorConfigArray) {
     csvOptions.headers = false;
@@ -2629,28 +2649,21 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
   try {
     for await (const row of import_stream2.Readable.from(csvStream)) {
       if (!isGeneratorConfigArray) {
-        const rowData = Object.keys(fileConfig.fields).reduce((acc, key) => {
+        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        for (const key of configKeys) {
           const keyGenerator = fileConfig.fields[key.trim()];
-          if (!keyGenerator) {
-            return acc;
-          }
-          return {
-            ...acc,
-            [keyGenerator.key]: keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key]
-          };
-        }, fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {});
+          if (!keyGenerator) continue;
+          rowData[keyGenerator.key] = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key];
+        }
         yield rowData;
       } else {
-        const rowData = Object.keys(row).reduce((acc, key, i) => {
+        const rowKeys = Object.keys(row);
+        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        for (let i = 0; i < rowKeys.length; i++) {
           const keyGenerator = fileConfig.fields[i];
-          if (!keyGenerator) {
-            return acc;
-          }
-          return {
-            ...acc,
-            [keyGenerator.key]: keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i]
-          };
-        }, fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {});
+          if (!keyGenerator) continue;
+          rowData[keyGenerator.key] = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i];
+        }
         yield rowData;
       }
     }
@@ -2658,6 +2671,17 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
     initialReadStream.destroy();
   }
 }
+async function countCsvLines(filePath) {
+  return new Promise((resolve, reject) => {
+    let newlines = 0;
+    (0, import_fs2.createReadStream)(filePath).on("data", (chunk2) => {
+      const buf = Buffer.isBuffer(chunk2) ? chunk2 : Buffer.from(chunk2);
+      for (let i = 0; i < buf.length; i++) {
+        if (buf[i] === 10) newlines++;
+      }
+    }).on("end", () => resolve(Math.max(0, newlines - 1))).on("error", reject);
+  });
+}
 var checkCsvHeaderRow = async (path2, fileConfig) => {
   return new Promise((resolve, reject) => {
     const colsWithParsingConfig = Object.keys(fileConfig.fields).map((key) => key.trim());
@@ -2851,8 +2875,19 @@ var FileStream = class extends Stream {
   /**
    * Yield records from all file(s).
    * When multiple file paths are provided, records are yielded sequentially from each file.
+   * Pre-counts total rows across all files to enable percentage progress logging.
    */
   async *_getRecords() {
+    let total = 0;
+    for (const filePath of this.localFilePaths) {
+      if (this.config.format === "EXCEL" /* EXCEL */) {
+        const data = await extractExcelRows(filePath, this.config.excel);
+        total += data.length;
+      } else if (this.config.format === "CSV" /* CSV */) {
+        total += await countCsvLines(filePath);
+      }
+    }
+    this.totalRows = total;
     for (const filePath of this.localFilePaths) {
       if (this.config.format === "EXCEL" /* EXCEL */) {
         const data = await extractExcelRows(filePath, this.config.excel);
@@ -3101,60 +3136,7 @@ var import_storage = require("@google-cloud/storage");
 var import_util5 = require("util");
 var pathModule = __toESM(require("path"));
 var import_fs3 = require("fs");
-
-// node_modules/uuid/dist/esm-node/rng.js
-var import_crypto = __toESM(require("crypto"));
-var rnds8Pool = new Uint8Array(256);
-var poolPtr = rnds8Pool.length;
-function rng() {
-  if (poolPtr > rnds8Pool.length - 16) {
-    import_crypto.default.randomFillSync(rnds8Pool);
-    poolPtr = 0;
-  }
-  return rnds8Pool.slice(poolPtr, poolPtr += 16);
-}
-
-// node_modules/uuid/dist/esm-node/regex.js
-var regex_default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i;
-
-// node_modules/uuid/dist/esm-node/validate.js
-function validate(uuid) {
-  return typeof uuid === "string" && regex_default.test(uuid);
-}
-var validate_default = validate;
-
-// node_modules/uuid/dist/esm-node/stringify.js
-var byteToHex = [];
-for (let i = 0; i < 256; ++i) {
-  byteToHex.push((i + 256).toString(16).substr(1));
-}
-function stringify3(arr, offset = 0) {
-  const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
-  if (!validate_default(uuid)) {
-    throw TypeError("Stringified UUID is invalid");
-  }
-  return uuid;
-}
-var stringify_default = stringify3;
-
-// node_modules/uuid/dist/esm-node/v4.js
-function v4(options, buf, offset) {
-  options = options || {};
-  const rnds = options.random || (options.rng || rng)();
-  rnds[6] = rnds[6] & 15 | 64;
-  rnds[8] = rnds[8] & 63 | 128;
-  if (buf) {
-    offset = offset || 0;
-    for (let i = 0; i < 16; ++i) {
-      buf[offset + i] = rnds[i];
-    }
-    return buf;
-  }
-  return stringify_default(rnds);
-}
-var v4_default = v4;
-
-// src/services/cloud-storage.ts
+var import_crypto = require("crypto");
 var logPrefix3 = "[CloudStorageService]";
 var tmpDir = "tmp";
 var CloudStorageService = class {
@@ -3292,12 +3274,15 @@ var CloudStorageService = class {
   }
   /**
    * Uploads a local file to the bucket with a unique name based on prefix, streamId, and UUID.
+   * Files are stored under `<prefix>/<run-id>/` when the RUN_ID env var is set,
+   * or `<prefix>/default/` otherwise.
    * Returns the GCS File reference.
    */
   async uploadFileWithUniqueName(filePath, prefix, streamId) {
     try {
       await this.bucket.get({ autoCreate: false });
-      const destinationFileName = `${prefix}/${streamId}-${v4_default()}.jsonnl`;
+      const runFolder = process.env.RUN_ID ?? "default";
+      const destinationFileName = `${prefix}/${runFolder}/${streamId}-${(0, import_crypto.randomUUID)()}.jsonnl`;
       await this.bucket.upload(filePath, {
         destination: destinationFileName
       });
@@ -3954,6 +3939,7 @@ var GCSStateProvider = class {
   _greaterThan,
   addWhalyFields,
   checkCsvHeaderRow,
+  countCsvLines,
   createCellReference,
   createCsvStreamConfig,
   createExcelGenerator,