@oronts/vendure-data-hub-plugin 0.1.0

package/dist/src/extractors/s3/client.d.ts

@@ -0,0 +1,59 @@
+/**
+ * S3 Client
+ *
+ * Manages S3 connections and operations using AWS SDK v3.
+ */
+import { ExtractorContext } from '../../types/index';
+import { S3ExtractorConfig, S3ObjectInfo } from './types';
+/**
+ * S3 client interface
+ */
+export interface S3Client {
+    listObjects(prefix?: string, continuationToken?: string): Promise<S3ListObjectsResult>;
+    getObject(key: string): Promise<Buffer>;
+    deleteObject(key: string): Promise<void>;
+    copyObject(sourceKey: string, destKey: string): Promise<void>;
+    headBucket(): Promise<boolean>;
+    close(): Promise<void>;
+}
+/**
+ * List objects result
+ */
+export interface S3ListObjectsResult {
+    objects: S3ObjectInfo[];
+    continuationToken?: string;
+    isTruncated: boolean;
+}
+/**
+ * Create S3 client configuration
+ */
+export interface S3ClientConfig {
+    region: string;
+    endpoint?: string;
+    forcePathStyle?: boolean;
+    credentials?: {
+        accessKeyId: string;
+        secretAccessKey: string;
+    };
+}
+/**
+ * Build S3 client configuration from extractor config
+ */
+export declare function buildS3ClientConfig(context: ExtractorContext, config: S3ExtractorConfig): Promise<S3ClientConfig>;
+/**
+ * Create S3 client
+ */
+export declare function createS3Client(context: ExtractorContext, config: S3ExtractorConfig): Promise<S3Client>;
+/**
+ * Test S3 connection by checking bucket access
+ */
+export declare function testS3Connection(context: ExtractorContext, config: S3ExtractorConfig): Promise<{
+    success: boolean;
+    error?: string;
+    latencyMs?: number;
+}>;
+/**
+ * Build source ID for S3 object
+ */
+export declare function buildS3SourceId(bucket: string, key: string): string;
+//# sourceMappingURL=client.d.ts.map

package/dist/src/extractors/s3/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../../src/extractors/s3/client.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAUH,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAe,MAAM,SAAS,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,QAAQ;IACrB,WAAW,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,iBAAiB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACvF,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9D,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;KAC3B,CAAC;CACL;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,gBAAgB,EACzB,MAAM,EAAE,iBAAiB,GAC1B,OAAO,CAAC,cAAc,CAAC,CAqBzB;AAED;;GAEG;AACH,wBAAsB,cAAc,CAChC,OAAO,EAAE,gBAAgB,EACzB,MAAM,EAAE,iBAAiB,GAC1B,OAAO,CAAC,QAAQ,CAAC,CA4FnB;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CAClC,OAAO,EAAE,gBAAgB,EACzB,MAAM,EAAE,iBAAiB,GAC1B,OAAO,CAAC;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAkBnE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAEnE"}

package/dist/src/extractors/s3/client.js

@@ -0,0 +1,147 @@
+"use strict";
+/**
+ * S3 Client
+ *
+ * Manages S3 connections and operations using AWS SDK v3.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildS3ClientConfig = buildS3ClientConfig;
+exports.createS3Client = createS3Client;
+exports.testS3Connection = testS3Connection;
+exports.buildS3SourceId = buildS3SourceId;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const error_utils_1 = require("../../utils/error.utils");
+const types_1 = require("./types");
+/**
+ * Build S3 client configuration from extractor config
+ */
+async function buildS3ClientConfig(context, config) {
+    const clientConfig = {
+        region: config.region || types_1.S3_DEFAULTS.region,
+        endpoint: config.endpoint,
+        forcePathStyle: config.forcePathStyle ?? types_1.S3_DEFAULTS.forcePathStyle,
+    };
+    // Resolve credentials if provided
+    if (config.accessKeyIdSecretCode && config.secretAccessKeySecretCode) {
+        const accessKeyId = await context.secrets.get(config.accessKeyIdSecretCode);
+        const secretAccessKey = await context.secrets.get(config.secretAccessKeySecretCode);
+        if (accessKeyId && secretAccessKey) {
+            clientConfig.credentials = {
+                accessKeyId,
+                secretAccessKey,
+            };
+        }
+    }
+    return clientConfig;
+}
+/**
+ * Create S3 client
+ */
+async function createS3Client(context, config) {
+    const clientConfig = await buildS3ClientConfig(context, config);
+    const s3 = new client_s3_1.S3Client({
+        region: clientConfig.region,
+        endpoint: clientConfig.endpoint,
+        forcePathStyle: clientConfig.forcePathStyle,
+        credentials: clientConfig.credentials,
+    });
+    const bucket = config.bucket;
+    return {
+        async listObjects(prefix, continuationToken) {
+            const command = new client_s3_1.ListObjectsV2Command({
+                Bucket: bucket,
+                Prefix: prefix || config.prefix,
+                ContinuationToken: continuationToken,
+                MaxKeys: types_1.S3_DEFAULTS.listMaxKeys,
+            });
+            const response = await s3.send(command);
+            return {
+                objects: (response.Contents || [])
+                    .filter(obj => obj.Key !== undefined)
+                    .map(obj => ({
+                    key: obj.Key,
+                    size: obj.Size || 0,
+                    lastModified: obj.LastModified || new Date(),
+                    etag: obj.ETag?.replace(/"/g, ''),
+                    storageClass: obj.StorageClass,
+                })),
+                continuationToken: response.NextContinuationToken,
+                isTruncated: response.IsTruncated || false,
+            };
+        },
+        async getObject(key) {
+            const command = new client_s3_1.GetObjectCommand({
+                Bucket: bucket,
+                Key: key,
+            });
+            const response = await s3.send(command);
+            if (response.Body) {
+                // AWS SDK v3 uses web streams
+                const body = response.Body;
+                if (typeof body.transformToByteArray === 'function') {
+                    return Buffer.from(await body.transformToByteArray());
+                }
+                // Fallback for readable streams
+                const chunks = [];
+                for await (const chunk of response.Body) {
+                    chunks.push(Buffer.from(chunk));
+                }
+                return Buffer.concat(chunks);
+            }
+            throw new Error(`Unable to read S3 object: ${key}`);
+        },
+        async deleteObject(key) {
+            const command = new client_s3_1.DeleteObjectCommand({
+                Bucket: bucket,
+                Key: key,
+            });
+            await s3.send(command);
+        },
+        async copyObject(sourceKey, destKey) {
+            const command = new client_s3_1.CopyObjectCommand({
+                Bucket: bucket,
+                CopySource: `${bucket}/${sourceKey}`,
+                Key: destKey,
+            });
+            await s3.send(command);
+        },
+        async headBucket() {
+            const command = new client_s3_1.HeadBucketCommand({
+                Bucket: bucket,
+            });
+            await s3.send(command);
+            return true;
+        },
+        async close() {
+            s3.destroy();
+        },
+    };
+}
+/**
+ * Test S3 connection by checking bucket access
+ */
+async function testS3Connection(context, config) {
+    const startTime = Date.now();
+    try {
+        const client = await createS3Client(context, config);
+        await client.headBucket();
+        await client.close();
+        return {
+            success: true,
+            latencyMs: Date.now() - startTime,
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: (0, error_utils_1.getErrorMessage)(error),
+        };
+    }
+}
+/**
+ * Build source ID for S3 object
+ */
+function buildS3SourceId(bucket, key) {
+    return `s3://${bucket}/${key}`;
+}
+//# sourceMappingURL=client.js.map

package/dist/src/extractors/s3/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../../src/extractors/s3/client.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;AAmDH,kDAwBC;AAKD,wCA+FC;AAKD,4CAqBC;AAKD,0CAEC;AA9MD,kDAO4B;AAE5B,yDAA0D;AAC1D,mCAAuE;AAoCvE;;GAEG;AACI,KAAK,UAAU,mBAAmB,CACrC,OAAyB,EACzB,MAAyB;IAEzB,MAAM,YAAY,GAAmB;QACjC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,mBAAW,CAAC,MAAM;QAC3C,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,mBAAW,CAAC,cAAc;KACtE,CAAC;IAEF,kCAAkC;IAClC,IAAI,MAAM,CAAC,qBAAqB,IAAI,MAAM,CAAC,yBAAyB,EAAE,CAAC;QACnE,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAC5E,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;QAEpF,IAAI,WAAW,IAAI,eAAe,EAAE,CAAC;YACjC,YAAY,CAAC,WAAW,GAAG;gBACvB,WAAW;gBACX,eAAe;aAClB,CAAC;QACN,CAAC;IACL,CAAC;IAED,OAAO,YAAY,CAAC;AACxB,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,cAAc,CAChC,OAAyB,EACzB,MAAyB;IAEzB,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEhE,MAAM,EAAE,GAAG,IAAI,oBAAW,CAAC;QACvB,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;QAC/B,cAAc,EAAE,YAAY,CAAC,cAAc;QAC3C,WAAW,EAAE,YAAY,CAAC,WAAW;KACxC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAE7B,OAAO;QACH,KAAK,CAAC,WAAW,CAAC,MAAe,EAAE,iBAA0B;YACzD,MAAM,OAAO,GAAG,IAAI,gCAAoB,CAAC;gBACrC,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,MAAM;gBAC/B,iBAAiB,EAAE,iBAAiB;gBACpC,OAAO,EAAE,mBAAW,CAAC,WAAW;aACnC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAExC,OAAO;gBACH,OAAO,EAAE,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;qBAC7B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC;qBACpC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACT,GAAG,EAAE,GAAG,CAAC,GAAa;oBACtB,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC;oBACnB,YAAY,EAAE,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE;oBAC5C,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;oBACjC,YAAY,EAAE,GAAG,CAAC,YAAY;iBACjC,CAAC,CAAC;gBACP,iBAAiB,EAAE,QAAQ,CAAC,qBAAqB;gBACjD,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,KAAK;aAC7C,CAAC;QACN,CAAC;QAED,KAAK,CAAC,SAAS,CAAC,GAAW;YACvB,MAAM,OAAO,GAAG,IAAI,4BAAgB,CAAC;gBACjC,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,GAAG;aACX,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAExC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAChB,8BAA8B;gBAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAA4D,CAAC;gBACnF,IAAI,OAAO,IAAI,CAAC,oBAAoB,KAAK,UAAU,EAAE,CAAC;oBAClD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC;gBAC1D,CAAC;gBACD,gCAAgC;gBAChC,MAAM,MAAM,GAAa,EAAE,CAAC;gBAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAAiC,EAAE,CAAC;oBACnE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACpC,CAAC;gBACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACjC,CAAC;YAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,KAAK,CAAC,YAAY,CAAC,GAAW;YAC1B,MAAM,OAAO,GAAG,IAAI,+BAAmB,CAAC;gBACpC,MAAM,EAAE,MAAM;gBACd,GAAG,EAAE,GAAG;aACX,CAAC,CAAC;YACH,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,SAAiB,EAAE,OAAe;YAC/C,MAAM,OAAO,GAAG,IAAI,6BAAiB,CAAC;gBAClC,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,GAAG,MAAM,IAAI,SAAS,EAAE;gBACpC,GAAG,EAAE,OAAO;aACf,CAAC,CAAC;YACH,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC;QAED,KAAK,CAAC,UAAU;YACZ,MAAM,OAAO,GAAG,IAAI,6BAAiB,CAAC;gBAClC,MAAM,EAAE,MAAM;aACjB,CAAC,CAAC;YACH,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,KAAK,CAAC,KAAK;YACP,EAAE,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;KACJ,CAAC;AACN,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,gBAAgB,CAClC,OAAyB,EACzB,MAAyB;IAEzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1B,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QAErB,OAAO;YACH,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACpC,CAAC;IACN,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO;YACH,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,IAAA,6BAAe,EAAC,KAAK,CAAC;SAChC,CAAC;IACN,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe,CAAC,MAAc,EAAE,GAAW;IACvD,OAAO,QAAQ,MAAM,IAAI,GAAG,EAAE,CAAC;AACnC,CAAC"}

package/dist/src/extractors/s3/file-handlers.d.ts

@@ -0,0 +1,54 @@
+/**
+ * S3 File Handlers
+ *
+ * Utilities for handling S3 object filtering, parsing, and post-processing.
+ */
+import { S3ObjectInfo, S3ExtractorConfig, S3ObjectMetadata } from './types';
+import { FileParserService } from '../../parsers/file-parser.service';
+import { JsonObject } from '../../types/index';
+import { detectFileFormat, parseModifiedAfterDate } from '../shared';
+/**
+ * Filter objects by suffix
+ */
+export declare function filterBySuffix(objects: S3ObjectInfo[], suffix?: string): S3ObjectInfo[];
+/**
+ * Filter objects by modification date.
+ * Uses consistent date filtering logic.
+ */
+export declare function filterByModifiedAfter(objects: S3ObjectInfo[], modifiedAfter?: string): S3ObjectInfo[];
+/**
+ * Filter objects based on configuration
+ */
+export declare function filterObjects(objects: S3ObjectInfo[], config: S3ExtractorConfig): S3ObjectInfo[];
+export { detectFileFormat };
+/**
+ * Parse S3 object content
+ */
+export declare function parseS3Content(content: Buffer, key: string, config: S3ExtractorConfig, fileParser: FileParserService): Promise<JsonObject[]>;
+/**
+ * Build object metadata for attaching to records
+ */
+export declare function buildObjectMetadata(bucket: string, obj: S3ObjectInfo): S3ObjectMetadata;
+/**
+ * Attach S3 metadata to record.
+ * Uses shared utility with S3-specific key.
+ */
+export declare function attachMetadataToRecord(record: JsonObject, metadata: S3ObjectMetadata): JsonObject;
+/**
+ * Calculate destination key for move operation
+ */
+export declare function calculateDestinationKey(sourceKey: string, sourcePrefix: string | undefined, destinationPrefix: string): string;
+/**
+ * Validate bucket name format
+ */
+export declare function isValidBucketName(name: string): boolean;
+/**
+ * Validate prefix format (should not start with /)
+ */
+export declare function isValidPrefix(prefix: string): boolean;
+export { parseModifiedAfterDate };
+/**
+ * Estimate total objects to process based on listing
+ */
+export declare function estimateObjectCount(objects: S3ObjectInfo[], isTruncated: boolean, maxObjects: number): number;
+//# sourceMappingURL=file-handlers.d.ts.map

package/dist/src/extractors/s3/file-handlers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-handlers.d.ts","sourceRoot":"","sources":["../../../../src/extractors/s3/file-handlers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACH,gBAAgB,EAEhB,sBAAsB,EAEzB,MAAM,WAAW,CAAC;AAEnB;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,YAAY,EAAE,CAGvF;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACjC,OAAO,EAAE,YAAY,EAAE,EACvB,aAAa,CAAC,EAAE,MAAM,GACvB,YAAY,EAAE,CAOhB;AAED;;GAEG;AACH,wBAAgB,aAAa,CACzB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,iBAAiB,GAC1B,YAAY,EAAE,CAahB;AAED,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B;;GAEG;AACH,wBAAsB,cAAc,CAChC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,iBAAiB,GAC9B,OAAO,CAAC,UAAU,EAAE,CAAC,CAavB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,YAAY,GAClB,gBAAgB,CAQlB;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAClC,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,gBAAgB,GAC3B,UAAU,CAEZ;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACnC,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GAAG,SAAS,EAChC,iBAAiB,EAAE,MAAM,GAC1B,MAAM,CAQR;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAoBvD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAErD;AAED,OAAO,EAAE,sBAAsB,EAAE,CAAC;AAElC;;GAEG;AACH,wBAAgB,mBAAmB,CAC/B,OAAO,EAAE,YAAY,EAAE,EACvB,WAAW,EAAE,OAAO,EACpB,UAAU,EAAE,MAAM,GACnB,MAAM,CAMR"}

package/dist/src/extractors/s3/file-handlers.js

@@ -0,0 +1,136 @@
+"use strict";
+/**
+ * S3 File Handlers
+ *
+ * Utilities for handling S3 object filtering, parsing, and post-processing.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseModifiedAfterDate = exports.detectFileFormat = void 0;
+exports.filterBySuffix = filterBySuffix;
+exports.filterByModifiedAfter = filterByModifiedAfter;
+exports.filterObjects = filterObjects;
+exports.parseS3Content = parseS3Content;
+exports.buildObjectMetadata = buildObjectMetadata;
+exports.attachMetadataToRecord = attachMetadataToRecord;
+exports.calculateDestinationKey = calculateDestinationKey;
+exports.isValidBucketName = isValidBucketName;
+exports.isValidPrefix = isValidPrefix;
+exports.estimateObjectCount = estimateObjectCount;
+const shared_1 = require("../shared");
+Object.defineProperty(exports, "detectFileFormat", { enumerable: true, get: function () { return shared_1.detectFileFormat; } });
+Object.defineProperty(exports, "parseModifiedAfterDate", { enumerable: true, get: function () { return shared_1.parseModifiedAfterDate; } });
+/**
+ * Filter objects by suffix
+ */
+function filterBySuffix(objects, suffix) {
+    if (!suffix)
+        return objects;
+    return objects.filter(obj => obj.key.endsWith(suffix));
+}
+/**
+ * Filter objects by modification date.
+ * Uses consistent date filtering logic.
+ */
+function filterByModifiedAfter(objects, modifiedAfter) {
+    if (!modifiedAfter)
+        return objects;
+    const afterDate = new Date(modifiedAfter);
+    if (isNaN(afterDate.getTime()))
+        return objects;
+    return objects.filter(obj => obj.lastModified >= afterDate);
+}
+/**
+ * Filter objects based on configuration
+ */
+function filterObjects(objects, config) {
+    let filtered = objects;
+    // Filter by suffix
+    filtered = filterBySuffix(filtered, config.suffix);
+    // Filter by modification date
+    filtered = filterByModifiedAfter(filtered, config.modifiedAfter);
+    // Filter out directories (keys ending with /)
+    filtered = filtered.filter(obj => !obj.key.endsWith('/'));
+    return filtered;
+}
+/**
+ * Parse S3 object content
+ */
+async function parseS3Content(content, key, config, fileParser) {
+    return (0, shared_1.parseFileContent)(content, key, {
+        format: config.format,
+        csv: config.csv,
+        json: config.json,
+        xml: config.xml,
+        xlsx: config.xlsx,
+    }, fileParser);
+}
+/**
+ * Build object metadata for attaching to records
+ */
+function buildObjectMetadata(bucket, obj) {
+    return {
+        bucket,
+        key: obj.key,
+        size: obj.size,
+        etag: obj.etag,
+        lastModified: obj.lastModified.toISOString(),
+    };
+}
+/**
+ * Attach S3 metadata to record.
+ * Uses shared utility with S3-specific key.
+ */
+function attachMetadataToRecord(record, metadata) {
+    return (0, shared_1.attachMetadataToRecord)(record, metadata, '_s3');
+}
+/**
+ * Calculate destination key for move operation
+ */
+function calculateDestinationKey(sourceKey, sourcePrefix, destinationPrefix) {
+    if (sourcePrefix && sourceKey.startsWith(sourcePrefix)) {
+        // Replace source prefix with destination prefix
+        return destinationPrefix + sourceKey.slice(sourcePrefix.length);
+    }
+    // Prepend destination prefix
+    return destinationPrefix + sourceKey.split('/').pop();
+}
+/**
+ * Validate bucket name format
+ */
+function isValidBucketName(name) {
+    // S3 bucket naming rules:
+    // - 3-63 characters
+    // - lowercase letters, numbers, hyphens, periods
+    // - must start and end with letter or number
+    // - no adjacent periods
+    // - not formatted as IP address
+    if (name.length < 3 || name.length > 63)
+        return false;
+    // Basic pattern check
+    if (!/^[a-z0-9][a-z0-9.-]*[a-z0-9]$/.test(name))
+        return false;
+    // No adjacent periods
+    if (/\.\./.test(name))
+        return false;
+    // Not an IP address
+    if (/^\d+\.\d+\.\d+\.\d+$/.test(name))
+        return false;
+    return true;
+}
+/**
+ * Validate prefix format (should not start with /)
+ */
+function isValidPrefix(prefix) {
+    return !prefix.startsWith('/');
+}
+/**
+ * Estimate total objects to process based on listing
+ */
+function estimateObjectCount(objects, isTruncated, maxObjects) {
+    if (!isTruncated) {
+        return Math.min(objects.length, maxObjects);
+    }
+    // If truncated, we don't know the exact count
+    return maxObjects;
+}
+//# sourceMappingURL=file-handlers.js.map

package/dist/src/extractors/s3/file-handlers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-handlers.js","sourceRoot":"","sources":["../../../../src/extractors/s3/file-handlers.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;AAeH,wCAGC;AAMD,sDAUC;AAKD,sCAgBC;AAOD,wCAkBC;AAKD,kDAWC;AAMD,wDAKC;AAKD,0DAYC;AAKD,8CAoBC;AAKD,sCAEC;AAOD,kDAUC;AAxKD,sCAKmB;AA+CV,iGAnDL,yBAAgB,OAmDK;AAqGhB,uGAtJL,+BAAsB,OAsJK;AAlJ/B;;GAEG;AACH,SAAgB,cAAc,CAAC,OAAuB,EAAE,MAAe;IACnE,IAAI,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAC5B,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,SAAgB,qBAAqB,CACjC,OAAuB,EACvB,aAAsB;IAEtB,IAAI,CAAC,aAAa;QAAE,OAAO,OAAO,CAAC;IAEnC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC;IAC1C,IAAI,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAAE,OAAO,OAAO,CAAC;IAE/C,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,SAAgB,aAAa,CACzB,OAAuB,EACvB,MAAyB;IAEzB,IAAI,QAAQ,GAAG,OAAO,CAAC;IAEvB,mBAAmB;IACnB,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAEnD,8BAA8B;IAC9B,QAAQ,GAAG,qBAAqB,CAAC,QAAQ,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;IAEjE,8CAA8C;IAC9C,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAE1D,OAAO,QAAQ,CAAC;AACpB,CAAC;AAID;;GAEG;AACI,KAAK,UAAU,cAAc,CAChC,OAAe,EACf,GAAW,EACX,MAAyB,EACzB,UAA6B;IAE7B,OAAO,IAAA,yBAAgB,EACnB,OAAO,EACP,GAAG,EACH;QACI,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,IAAI,EAAE,MAAM,CAAC,IAAI;KACpB,EACD,UAAU,CACb,CAAC;AACN,CAAC;AAED;;GAEG;AACH,SAAgB,mBAAmB,CAC/B,MAAc,EACd,GAAiB;IAEjB,OAAO;QACH,MAAM;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,YAAY,EAAE,GAAG,CAAC,YAAY,CAAC,WAAW,EAAE;KAC/C,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,SAAgB,sBAAsB,CAClC,MAAkB,EAClB,QAA0B;IAE1B,OAAO,IAAA,+BAA4B,EAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACjE,CAAC;AAED;;GAEG;AACH,SAAgB,uBAAuB,CACnC,SAAiB,EACjB,YAAgC,EAChC,iBAAyB;IAEzB,IAAI,YAAY,IAAI,SAAS,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACrD,gDAAgD;QAChD,OAAO,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACpE,CAAC;IAED,6BAA6B;IAC7B,OAAO,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB,CAAC,IAAY;IAC1C,0BAA0B;IAC1B,oBAAoB;IACpB,iDAAiD;IACjD,6CAA6C;IAC7C,wBAAwB;IACxB,gCAAgC;IAEhC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,KAAK,CAAC;IAEtD,sBAAsB;IACtB,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE9D,sBAAsB;IACtB,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEpC,oBAAoB;IACpB,IAAI,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEpD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAgB,aAAa,CAAC,MAAc;IACxC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;AACnC,CAAC;AAID;;GAEG;AACH,SAAgB,mBAAmB,CAC/B,OAAuB,EACvB,WAAoB,EACpB,UAAkB;IAElB,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAChD,CAAC;IACD,8CAA8C;IAC9C,OAAO,UAAU,CAAC;AACtB,CAAC"}

package/dist/src/extractors/s3/index.d.ts

@@ -0,0 +1,5 @@
+export * from './types';
+export { S3Client, S3ListObjectsResult, S3ClientConfig, createS3Client, buildS3ClientConfig, testS3Connection, buildS3SourceId, } from './client';
+export { filterBySuffix, filterByModifiedAfter, filterObjects, detectFileFormat, parseS3Content, buildObjectMetadata, attachMetadataToRecord, calculateDestinationKey, isValidBucketName, isValidPrefix, parseModifiedAfterDate, estimateObjectCount, } from './file-handlers';
+export { S3Extractor } from './s3.extractor';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/extractors/s3/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/extractors/s3/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AAExB,OAAO,EACH,QAAQ,EACR,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,gBAAgB,EAChB,eAAe,GAClB,MAAM,UAAU,CAAC;AAElB,OAAO,EACH,cAAc,EACd,qBAAqB,EACrB,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,uBAAuB,EACvB,iBAAiB,EACjB,aAAa,EACb,sBAAsB,EACtB,mBAAmB,GACtB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/src/extractors/s3/index.js

@@ -0,0 +1,39 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3Extractor = exports.estimateObjectCount = exports.parseModifiedAfterDate = exports.isValidPrefix = exports.isValidBucketName = exports.calculateDestinationKey = exports.attachMetadataToRecord = exports.buildObjectMetadata = exports.parseS3Content = exports.detectFileFormat = exports.filterObjects = exports.filterByModifiedAfter = exports.filterBySuffix = exports.buildS3SourceId = exports.testS3Connection = exports.buildS3ClientConfig = exports.createS3Client = void 0;
+__exportStar(require("./types"), exports);
+var client_1 = require("./client");
+Object.defineProperty(exports, "createS3Client", { enumerable: true, get: function () { return client_1.createS3Client; } });
+Object.defineProperty(exports, "buildS3ClientConfig", { enumerable: true, get: function () { return client_1.buildS3ClientConfig; } });
+Object.defineProperty(exports, "testS3Connection", { enumerable: true, get: function () { return client_1.testS3Connection; } });
+Object.defineProperty(exports, "buildS3SourceId", { enumerable: true, get: function () { return client_1.buildS3SourceId; } });
+var file_handlers_1 = require("./file-handlers");
+Object.defineProperty(exports, "filterBySuffix", { enumerable: true, get: function () { return file_handlers_1.filterBySuffix; } });
+Object.defineProperty(exports, "filterByModifiedAfter", { enumerable: true, get: function () { return file_handlers_1.filterByModifiedAfter; } });
+Object.defineProperty(exports, "filterObjects", { enumerable: true, get: function () { return file_handlers_1.filterObjects; } });
+Object.defineProperty(exports, "detectFileFormat", { enumerable: true, get: function () { return file_handlers_1.detectFileFormat; } });
+Object.defineProperty(exports, "parseS3Content", { enumerable: true, get: function () { return file_handlers_1.parseS3Content; } });
+Object.defineProperty(exports, "buildObjectMetadata", { enumerable: true, get: function () { return file_handlers_1.buildObjectMetadata; } });
+Object.defineProperty(exports, "attachMetadataToRecord", { enumerable: true, get: function () { return file_handlers_1.attachMetadataToRecord; } });
+Object.defineProperty(exports, "calculateDestinationKey", { enumerable: true, get: function () { return file_handlers_1.calculateDestinationKey; } });
+Object.defineProperty(exports, "isValidBucketName", { enumerable: true, get: function () { return file_handlers_1.isValidBucketName; } });
+Object.defineProperty(exports, "isValidPrefix", { enumerable: true, get: function () { return file_handlers_1.isValidPrefix; } });
+Object.defineProperty(exports, "parseModifiedAfterDate", { enumerable: true, get: function () { return file_handlers_1.parseModifiedAfterDate; } });
+Object.defineProperty(exports, "estimateObjectCount", { enumerable: true, get: function () { return file_handlers_1.estimateObjectCount; } });
+var s3_extractor_1 = require("./s3.extractor");
+Object.defineProperty(exports, "S3Extractor", { enumerable: true, get: function () { return s3_extractor_1.S3Extractor; } });
+//# sourceMappingURL=index.js.map

package/dist/src/extractors/s3/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/extractors/s3/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,0CAAwB;AAExB,mCAQkB;AAJd,wGAAA,cAAc,OAAA;AACd,6GAAA,mBAAmB,OAAA;AACnB,0GAAA,gBAAgB,OAAA;AAChB,yGAAA,eAAe,OAAA;AAGnB,iDAayB;AAZrB,+GAAA,cAAc,OAAA;AACd,sHAAA,qBAAqB,OAAA;AACrB,8GAAA,aAAa,OAAA;AACb,iHAAA,gBAAgB,OAAA;AAChB,+GAAA,cAAc,OAAA;AACd,oHAAA,mBAAmB,OAAA;AACnB,uHAAA,sBAAsB,OAAA;AACtB,wHAAA,uBAAuB,OAAA;AACvB,kHAAA,iBAAiB,OAAA;AACjB,8GAAA,aAAa,OAAA;AACb,uHAAA,sBAAsB,OAAA;AACtB,oHAAA,mBAAmB,OAAA;AAGvB,+CAA6C;AAApC,2GAAA,WAAW,OAAA"}

package/dist/src/extractors/s3/s3.extractor.d.ts

@@ -0,0 +1,20 @@
+import { DataExtractor, ExtractorContext, ExtractorValidationResult, ConnectionTestResult, ExtractorPreviewResult, RecordEnvelope, StepConfigSchema, ExtractorCategory } from '../../types/index';
+import { FileParserService } from '../../parsers/file-parser.service';
+import { S3ExtractorConfig } from './types';
+export declare class S3Extractor implements DataExtractor<S3ExtractorConfig> {
+    private readonly fileParser;
+    readonly type: "EXTRACTOR";
+    readonly code = "s3";
+    readonly name = "S3 Extractor";
+    readonly category: ExtractorCategory;
+    readonly supportsPagination = true;
+    readonly supportsIncremental = true;
+    readonly supportsCancellation = true;
+    constructor(fileParser: FileParserService);
+    readonly schema: StepConfigSchema;
+    extract(context: ExtractorContext, config: S3ExtractorConfig): AsyncGenerator<RecordEnvelope, void, undefined>;
+    validate(_context: ExtractorContext, config: S3ExtractorConfig): Promise<ExtractorValidationResult>;
+    testConnection(context: ExtractorContext, config: S3ExtractorConfig): Promise<ConnectionTestResult>;
+    preview(context: ExtractorContext, config: S3ExtractorConfig, limit?: number): Promise<ExtractorPreviewResult>;
+}
+//# sourceMappingURL=s3.extractor.d.ts.map

package/dist/src/extractors/s3/s3.extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3.extractor.d.ts","sourceRoot":"","sources":["../../../../src/extractors/s3/s3.extractor.ts"],"names":[],"mappings":"AACA,OAAO,EACH,aAAa,EACb,gBAAgB,EAChB,yBAAyB,EACzB,oBAAoB,EACpB,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AAItE,OAAO,EACH,iBAAiB,EAEpB,MAAM,SAAS,CAAC;AAoBjB,qBACa,WAAY,YAAW,aAAa,CAAC,iBAAiB,CAAC;IASpD,OAAO,CAAC,QAAQ,CAAC,UAAU;IARvC,QAAQ,CAAC,IAAI,EAAG,WAAW,CAAU;IACrC,QAAQ,CAAC,IAAI,QAAQ;IACrB,QAAQ,CAAC,IAAI,kBAAkB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAmB;IACvD,QAAQ,CAAC,kBAAkB,QAAQ;IACnC,QAAQ,CAAC,mBAAmB,QAAQ;IACpC,QAAQ,CAAC,oBAAoB,QAAQ;gBAER,UAAU,EAAE,iBAAiB;IAE1D,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAuB;IAEjD,OAAO,CACV,OAAO,EAAE,gBAAgB,EACzB,MAAM,EAAE,iBAAiB,GAC1B,cAAc,CAAC,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC;IAyF5C,QAAQ,CACV,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,iBAAiB,GAC1B,OAAO,CAAC,yBAAyB,CAAC;IAmE/B,cAAc,CAChB,OAAO,EAAE,gBAAgB,EACzB,MAAM,EAAE,iBAAiB,GAC1B,OAAO,CAAC,oBAAoB,CAAC;IA2B1B,OAAO,CACT,OAAO,EAAE,gBAAgB,EACzB,MAAM,EAAE,iBAAiB,EACzB,KAAK,GAAE,MAAW,GACnB,OAAO,CAAC,sBAAsB,CAAC;CAqDrC"}