@whaly/connector-sdk 0.3.1 → 0.3.3

package/dist/index.d.mts

@@ -187,10 +187,12 @@ interface StreamWithTempFile {
     streamId: StreamId;
     tempFile: TempFile;
 }
+declare const DEFAULT_SYNCED_AT_COLUMN = "_wly_synced_at";
 interface BaseConfig {
     connector_id: string;
     database: string;
     schema: string;
+    syncedAtColumnName?: string;
 }
 
 interface ColumnMappingStoreUnsafeToSafe {
@@ -302,6 +304,7 @@ interface StateProvider {
 
 declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     config: C;
+    syncedAtColumnName: string;
     schemaHooks: TargetSchemaHook[];
     syncTime: Dayjs;
     stateProvider: StateProvider;
@@ -612,7 +615,7 @@ declare class MissingSchemaError extends Error {
 }
 
 declare const removeParasiteProperties: (record: any, schema: FlattenedSchema) => any;
-declare const addWhalyFields: (record: any, batchDate: string) => any;
+declare const addWhalyFields: (record: any, batchDate: string, columnName?: string) => any;
 
 declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSchema;
 
@@ -682,7 +685,6 @@ interface CsvFileConfig {
     encoding?: string;
     separator: string;
     fields: CsvFieldsConfig;
-    addSyncedAtColumn?: boolean;
 }
 interface FileStreamBaseConfig {
     streamId: string;
@@ -765,7 +767,7 @@ declare const createCellReference: (columnIndex: number, rowIndex: number) => st
 /**
  * Low-level Excel extractor: returns array of row records from a workbook.
  */
-declare const extractExcelRows: (localFilePath: string, conf: ExcelExtractionConfig) => Promise<Record<string, string>[]>;
+declare const extractExcelRows: (localFilePath: string, conf: ExcelExtractionConfig) => Promise<Record<string, string | null>[]>;
 /**
  * Validates an Excel file configuration.
  */
@@ -777,7 +779,7 @@ declare const findAllMatchingConfigs: (filename: string, configs: ExcelExtractio
 /**
  * Creates an async generator from Excel data.
  */
-declare function createExcelGenerator(data: Record<string, string>[]): AsyncGenerator<Record<string, any>, void, unknown>;
+declare function createExcelGenerator(data: Record<string, string | null>[]): AsyncGenerator<Record<string, any>, void, unknown>;
 
 /**
  * Builder pattern for creating Excel configurations.
@@ -839,7 +841,6 @@ declare const writeGeneratorToCSV: (generator: AsyncGenerator, outputFileName: s
  * const config = CsvExtractionConfigBuilder.create()
  *   .separator(";")
  *   .encoding("latin1")
- *   .addSyncedAtColumn()
  *   .fieldsFromDict({
  *     "Product ID":   { key: "product_id",   type: "STRING" },
  *     "Product Name": { key: "product_name", type: "STRING" },
@@ -851,7 +852,6 @@ declare const writeGeneratorToCSV: (generator: AsyncGenerator, outputFileName: s
 declare class CsvExtractionConfigBuilder {
     private _separator;
     private _encoding?;
-    private _addSyncedAtColumn?;
     private _fields?;
     private _fieldsMode?;
     private _replicationMethod?;
@@ -859,7 +859,6 @@ declare class CsvExtractionConfigBuilder {
     static create(): CsvExtractionConfigBuilder;
     separator(sep: string): CsvExtractionConfigBuilder;
     encoding(enc: string): CsvExtractionConfigBuilder;
-    addSyncedAtColumn(enabled?: boolean): CsvExtractionConfigBuilder;
     fieldsFromDict(mapping: {
         [header: string]: {
             key: string;
@@ -1143,4 +1142,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };

package/dist/index.d.ts

@@ -187,10 +187,12 @@ interface StreamWithTempFile {
     streamId: StreamId;
     tempFile: TempFile;
 }
+declare const DEFAULT_SYNCED_AT_COLUMN = "_wly_synced_at";
 interface BaseConfig {
     connector_id: string;
     database: string;
     schema: string;
+    syncedAtColumnName?: string;
 }
 
 interface ColumnMappingStoreUnsafeToSafe {
@@ -302,6 +304,7 @@ interface StateProvider {
 
 declare abstract class ITarget<C extends BaseConfig = BaseConfig> {
     config: C;
+    syncedAtColumnName: string;
     schemaHooks: TargetSchemaHook[];
     syncTime: Dayjs;
     stateProvider: StateProvider;
@@ -612,7 +615,7 @@ declare class MissingSchemaError extends Error {
 }
 
 declare const removeParasiteProperties: (record: any, schema: FlattenedSchema) => any;
-declare const addWhalyFields: (record: any, batchDate: string) => any;
+declare const addWhalyFields: (record: any, batchDate: string, columnName?: string) => any;
 
 declare const flattenSchema: <T>(streamId: StreamId, schema: T) => FlattenedSchema;
 
@@ -682,7 +685,6 @@ interface CsvFileConfig {
     encoding?: string;
     separator: string;
     fields: CsvFieldsConfig;
-    addSyncedAtColumn?: boolean;
 }
 interface FileStreamBaseConfig {
     streamId: string;
@@ -765,7 +767,7 @@ declare const createCellReference: (columnIndex: number, rowIndex: number) => st
 /**
  * Low-level Excel extractor: returns array of row records from a workbook.
  */
-declare const extractExcelRows: (localFilePath: string, conf: ExcelExtractionConfig) => Promise<Record<string, string>[]>;
+declare const extractExcelRows: (localFilePath: string, conf: ExcelExtractionConfig) => Promise<Record<string, string | null>[]>;
 /**
  * Validates an Excel file configuration.
  */
@@ -777,7 +779,7 @@ declare const findAllMatchingConfigs: (filename: string, configs: ExcelExtractio
 /**
  * Creates an async generator from Excel data.
  */
-declare function createExcelGenerator(data: Record<string, string>[]): AsyncGenerator<Record<string, any>, void, unknown>;
+declare function createExcelGenerator(data: Record<string, string | null>[]): AsyncGenerator<Record<string, any>, void, unknown>;
 
 /**
  * Builder pattern for creating Excel configurations.
@@ -839,7 +841,6 @@ declare const writeGeneratorToCSV: (generator: AsyncGenerator, outputFileName: s
  * const config = CsvExtractionConfigBuilder.create()
  *   .separator(";")
  *   .encoding("latin1")
- *   .addSyncedAtColumn()
  *   .fieldsFromDict({
  *     "Product ID":   { key: "product_id",   type: "STRING" },
  *     "Product Name": { key: "product_name", type: "STRING" },
@@ -851,7 +852,6 @@ declare const writeGeneratorToCSV: (generator: AsyncGenerator, outputFileName: s
 declare class CsvExtractionConfigBuilder {
     private _separator;
     private _encoding?;
-    private _addSyncedAtColumn?;
     private _fields?;
     private _fieldsMode?;
     private _replicationMethod?;
@@ -859,7 +859,6 @@ declare class CsvExtractionConfigBuilder {
     static create(): CsvExtractionConfigBuilder;
     separator(sep: string): CsvExtractionConfigBuilder;
     encoding(enc: string): CsvExtractionConfigBuilder;
-    addSyncedAtColumn(enabled?: boolean): CsvExtractionConfigBuilder;
     fieldsFromDict(mapping: {
         [header: string]: {
             key: string;
@@ -1143,4 +1142,4 @@ declare class GCSStateProvider implements StateProvider {
     writeState(state: string): Promise<void>;
 }
 
-export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };
+export { ALL_STREAM_ID_KEYWORD, API_CALLS_METRIC_NAME, Authenticator, BATCH_INTERVAL_MS, type BaseConfig, type BigQueryConfig, BigQueryTarget, type BigqueryLocation, type Bookmark, type Bookmarks, CloudStorageService, type CloudStorageServiceOptions, type ColumnMappingStoreUnsafeToSafe, CounterMetric, CsvExtractionConfigBuilder, type CsvFieldsArrayConfig, type CsvFieldsConfig, type CsvFieldsDictConfig, type CsvFileConfig, type CsvStreamConfig, DEFAULT_MAX_CONCURRENT_STREAMS, DEFAULT_SYNCED_AT_COLUMN, EXECUTION_TIME_METRIC_NAME, type ErrorType, type ExcelCustomExtractorConfig, type ExcelDerivedField, type ExcelExtractionBaseConfig, type ExcelExtractionConfig, ExcelExtractionConfigBuilder, type ExcelFieldMapping, type ExcelFieldSpec, type ExcelSingleSheetExtractionConfig, type ExcelSourceColumn, type ExcelStreamConfig, type FieldType, FileFormat, FilePatterns, FileStream, type FileStreamConfig, type FileStreamEntry, FileTap, type FileTapConfig, type FlattenedSchema, GCSStateProvider, type GetConfig, type HTTPHeaders, type HTTPMethod, ITarget, type InputTapState, type InternalTableField, type JSONSchemaFieldDefinition, type Message, type MessageType, type MetricConfiguration, MissingFieldInSchemaError, MissingSchemaError, type PostConfig, type ProgressMarkers, type PropertiesMetadata, RESTStream, ROWS_SYNCED_METRIC_NAME, RecordMessage, RenameColumnStore, ReplicationMethod, ReplicationMethodMessage, type SafeColumnNameConverterFn, type Schema, SchemaMessage, SchemaValidationError, SftpService, type State, type StateHolder, StateMessage, type StateProvider, StateService, Stream, type StreamId, type StreamState, StreamWarehouseSyncService, type StreamWithTempFile, type SyncOptions, Tap, type TapState, type TargetSchemaHook, type TargetSchemaHookInput, type TempFile, type URLParams, VariableExtractors, type WarehouseTableField, _greaterThan, addWhalyFields, checkCsvHeaderRow, countCsvLines, createCellReference, createCsvStreamConfig, createExcelGenerator, createExcelStreamConfig, createTemporaryFileStream, csvFieldsToJsonSchema, excelColumnToIndex, excelFieldsToJsonSchema, excelRowToIndex, extractExcelRows, extractPrimaryKeysFromCsvConfig, extractPrimaryKeysFromExcelFields, extractStateForStream, fieldTypeToJsonSchema, finalizeStateProgressMarkers, findAllMatchingConfigs, flattenSchema, getAllMetrics, getAxiosInstance, getCounterMetric, getCounterMetrics, getDownloadFileApiCall, getJSONApiCall, getJSONApiCallWithFullResponse, getRawJSONApiCall, gracefulExit, haltAndCatchFire, incrementStreamState, indexToExcelColumn, indexToExcelRow, loadJson, loadSchema, logger, parseCellReference, postFormDataApiCall, postJSONApiCall, postUrlEncodedApiCall, printMemoryFootprint, processFileStreams, removeParasiteProperties, rowGeneratorFromCsv, safePath, unzip, validateExcelConfig, writeDataToCsv, writeGeneratorToCSV };

package/dist/index.js

@@ -39,6 +39,7 @@ __export(index_exports, {
   CounterMetric: () => CounterMetric,
   CsvExtractionConfigBuilder: () => CsvExtractionConfigBuilder,
   DEFAULT_MAX_CONCURRENT_STREAMS: () => DEFAULT_MAX_CONCURRENT_STREAMS,
+  DEFAULT_SYNCED_AT_COLUMN: () => DEFAULT_SYNCED_AT_COLUMN,
   EXECUTION_TIME_METRIC_NAME: () => EXECUTION_TIME_METRIC_NAME,
   ExcelExtractionConfigBuilder: () => ExcelExtractionConfigBuilder,
   FileFormat: () => FileFormat,
@@ -1670,6 +1671,9 @@ var MissingSchemaError = class extends Error {
   }
 };
 
+// src/sdk/models/target/models.ts
+var DEFAULT_SYNCED_AT_COLUMN = "_wly_synced_at";
+
 // src/sdk/models/target/record.ts
 var removeParasiteProperties = (record, schema) => {
   const schemaKeys = Object.keys(schema);
@@ -1680,10 +1684,10 @@ var removeParasiteProperties = (record, schema) => {
   });
   return record;
 };
-var addWhalyFields = (record, batchDate) => {
+var addWhalyFields = (record, batchDate, columnName = DEFAULT_SYNCED_AT_COLUMN) => {
   return {
     ...record,
-    _whaly_synced: batchDate
+    [columnName]: batchDate
   };
 };
 
@@ -1952,6 +1956,7 @@ var import_bluebird3 = __toESM(require("bluebird"));
 var semaphore = new import_async_mutex.Semaphore(1);
 var ITarget = class _ITarget {
   config;
+  syncedAtColumnName;
   schemaHooks;
   syncTime;
   stateProvider;
@@ -1967,6 +1972,7 @@ var ITarget = class _ITarget {
   ajv;
   constructor(config, stateProvider) {
     this.config = config;
+    this.syncedAtColumnName = config.syncedAtColumnName ?? DEFAULT_SYNCED_AT_COLUMN;
     this.stateProvider = stateProvider;
     this.schemaHooks = [];
     this.streams = {};
@@ -1987,8 +1993,8 @@ var ITarget = class _ITarget {
   analyzeSchemaChanges = (previousSchema, newSchema, streamId) => {
     const changes = [];
     let hasBreakingChanges = false;
-    const prevColumns = new Set(Object.keys(previousSchema).filter((col) => !col.startsWith("_whaly")));
-    const newColumns = new Set(Object.keys(newSchema).filter((col) => !col.startsWith("_whaly")));
+    const prevColumns = new Set(Object.keys(previousSchema).filter((col) => col !== this.syncedAtColumnName));
+    const newColumns = new Set(Object.keys(newSchema).filter((col) => col !== this.syncedAtColumnName));
     for (const columnName of newColumns) {
       if (!prevColumns.has(columnName)) {
         changes.push(`Added column '${columnName}'`);
@@ -2046,7 +2052,7 @@ var ITarget = class _ITarget {
         return;
       }
       const recordWithoutParasiteProperties = removeParasiteProperties(record, schema);
-      const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate());
+      const recordWithWhalyFields = addWhalyFields(recordWithoutParasiteProperties, stream.getBatchDate(), this.syncedAtColumnName);
       const validateFn = stream.getCompiledValidateFn();
       if (validateFn && !validateFn(recordWithWhalyFields)) {
         throw new SchemaValidationError(`Stream: ${streamId} - Record is not valid according to schema.
@@ -2129,7 +2135,7 @@ var ITarget = class _ITarget {
         throw new MissingSchemaError(`Stream: ${streamId} - Received SCHEMA message before stream state was initialized.`, streamId);
       }
       const schema = flattenSchema(streamId, message.schema);
-      schema["_whaly_synced"] = {
+      schema[this.syncedAtColumnName] = {
         format: "date-time",
         type: ["null", "string"]
       };
@@ -2425,10 +2431,10 @@ var extractSingleSheetRows = async (fileName, conf) => {
         if (!variables) {
           throw new Error(`No variables extracted from filename, cannot extract derived field ${key}`);
         }
-        acc[key] = variables[column.variableName] || "";
+        acc[key] = variables[column.variableName] || null;
       } else {
         const colIndex = excelColumnToIndex(column.column);
-        acc[key] = row[colIndex]?.v || "";
+        acc[key] = row[colIndex]?.v || null;
       }
       return acc;
     }, {});
@@ -2634,7 +2640,6 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
     csvOptions.headers = false;
   }
   const csvStream = (0, import_csv_parser.default)(csvOptions);
-  const syncedAt = (/* @__PURE__ */ new Date()).toISOString();
   let initialReadStream;
   let finalInputStream;
   const encodingToUse = fileConfig.encoding ? fileConfig.encoding.toLowerCase() : null;
@@ -2649,20 +2654,22 @@ async function* rowGeneratorFromCsv(path2, fileConfig) {
   try {
     for await (const row of import_stream2.Readable.from(csvStream)) {
       if (!isGeneratorConfigArray) {
-        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        const rowData = {};
         for (const key of configKeys) {
           const keyGenerator = fileConfig.fields[key.trim()];
           if (!keyGenerator) continue;
-          rowData[keyGenerator.key] = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key];
+          const rawValue = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[key]) : row[key];
+          rowData[keyGenerator.key] = rawValue === "" ? null : rawValue;
         }
         yield rowData;
       } else {
         const rowKeys = Object.keys(row);
-        const rowData = fileConfig.addSyncedAtColumn ? { _wly_synced_at: syncedAt } : {};
+        const rowData = {};
         for (let i = 0; i < rowKeys.length; i++) {
           const keyGenerator = fileConfig.fields[i];
           if (!keyGenerator) continue;
-          rowData[keyGenerator.key] = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i];
+          const rawValue = keyGenerator.valueTransformer ? keyGenerator.valueTransformer(row[i]) : row[i];
+          rowData[keyGenerator.key] = rawValue === "" ? null : rawValue;
         }
         yield rowData;
       }
@@ -2960,7 +2967,6 @@ async function processFileStreams(entries, tapState, target) {
 var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
   _separator = ",";
   _encoding;
-  _addSyncedAtColumn;
   _fields;
   _fieldsMode;
   _replicationMethod;
@@ -2976,10 +2982,6 @@ var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
     this._encoding = enc;
     return this;
   }
-  addSyncedAtColumn(enabled = true) {
-    this._addSyncedAtColumn = enabled;
-    return this;
-  }
   fieldsFromDict(mapping) {
     if (this._fieldsMode === "array") {
       throw new Error("Cannot use fieldsFromDict after fieldsFromArray");
@@ -3021,9 +3023,6 @@ var CsvExtractionConfigBuilder = class _CsvExtractionConfigBuilder {
     if (this._encoding !== void 0) {
       config.encoding = this._encoding;
     }
-    if (this._addSyncedAtColumn !== void 0) {
-      config.addSyncedAtColumn = this._addSyncedAtColumn;
-    }
     return config;
   }
   buildStreamConfig(streamId) {
@@ -3910,6 +3909,7 @@ var GCSStateProvider = class {
   CounterMetric,
   CsvExtractionConfigBuilder,
   DEFAULT_MAX_CONCURRENT_STREAMS,
+  DEFAULT_SYNCED_AT_COLUMN,
   EXECUTION_TIME_METRIC_NAME,
   ExcelExtractionConfigBuilder,
   FileFormat,