@objectstack/spec 0.3.3 → 0.4.2

package/dist/automation/etl.zod.js

@@ -0,0 +1,359 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ETL = exports.ETLPipelineRunSchema = exports.ETLRunStatusSchema = exports.ETLPipelineSchema = exports.ETLSyncModeSchema = exports.ETLTransformationSchema = exports.ETLTransformationTypeSchema = exports.ETLDestinationSchema = exports.ETLSourceSchema = exports.ETLEndpointTypeSchema = void 0;
+const zod_1 = require("zod");
+/**
+ * ETL (Extract, Transform, Load) Pipeline Protocol
+ *
+ * Inspired by modern data integration platforms like Airbyte, Fivetran, and Apache NiFi.
+ *
+ * ETL pipelines enable automated data synchronization between systems, transforming
+ * data as it moves from source to destination.
+ *
+ * ## Use Cases
+ *
+ * 1. **Data Warehouse Population**
+ *    - Extract from multiple operational systems
+ *    - Transform to analytical schema
+ *    - Load into data warehouse
+ *
+ * 2. **System Integration**
+ *    - Sync data between CRM and Marketing Automation
+ *    - Keep product catalog synchronized across e-commerce platforms
+ *    - Replicate data for backup/disaster recovery
+ *
+ * 3. **Data Migration**
+ *    - Move data from legacy systems to modern platforms
+ *    - Consolidate data from multiple sources
+ *    - Split monolithic databases into microservices
+ *
+ * @see https://airbyte.com/
+ * @see https://docs.fivetran.com/
+ * @see https://nifi.apache.org/
+ *
+ * @example
+ * ```typescript
+ * const salesforceToDB: ETLPipeline = {
+ *   name: 'salesforce_to_postgres',
+ *   label: 'Salesforce Accounts to PostgreSQL',
+ *   source: {
+ *     type: 'api',
+ *     connector: 'salesforce',
+ *     config: { object: 'Account' }
+ *   },
+ *   destination: {
+ *     type: 'database',
+ *     connector: 'postgres',
+ *     config: { table: 'accounts' }
+ *   },
+ *   transformations: [
+ *     { type: 'map', config: { 'Name': 'account_name' } }
+ *   ],
+ *   schedule: '0 2 * * *' // Daily at 2 AM
+ * }
+ * ```
+ */
+/**
+ * ETL Source/Destination Type
+ */
+exports.ETLEndpointTypeSchema = zod_1.z.enum([
+    'database', // SQL/NoSQL databases
+    'api', // REST/GraphQL APIs
+    'file', // CSV, JSON, XML, Excel files
+    'stream', // Kafka, RabbitMQ, Kinesis
+    'object', // ObjectStack object
+    'warehouse', // Data warehouse (Snowflake, BigQuery, Redshift)
+    'storage', // S3, Azure Blob, Google Cloud Storage
+    'spreadsheet', // Google Sheets, Excel Online
+]);
+/**
+ * ETL Source Configuration
+ */
+exports.ETLSourceSchema = zod_1.z.object({
+    /**
+     * Source type
+     */
+    type: exports.ETLEndpointTypeSchema.describe('Source type'),
+    /**
+     * Connector identifier
+     * References a registered connector
+     *
+     * @example "salesforce", "postgres", "mysql", "s3"
+     */
+    connector: zod_1.z.string().optional().describe('Connector ID'),
+    /**
+     * Source-specific configuration
+     * Structure varies by source type
+     *
+     * @example For database: { table: 'customers', schema: 'public' }
+     * @example For API: { endpoint: '/api/users', method: 'GET' }
+     * @example For file: { path: 's3://bucket/data.csv', format: 'csv' }
+     */
+    config: zod_1.z.record(zod_1.z.any()).describe('Source configuration'),
+    /**
+     * Incremental sync configuration
+     * Allows extracting only changed data
+     */
+    incremental: zod_1.z.object({
+        enabled: zod_1.z.boolean().default(false),
+        cursorField: zod_1.z.string().describe('Field to track progress (e.g., updated_at)'),
+        cursorValue: zod_1.z.any().optional().describe('Last processed value'),
+    }).optional().describe('Incremental extraction config'),
+});
+/**
+ * ETL Destination Configuration
+ */
+exports.ETLDestinationSchema = zod_1.z.object({
+    /**
+     * Destination type
+     */
+    type: exports.ETLEndpointTypeSchema.describe('Destination type'),
+    /**
+     * Connector identifier
+     */
+    connector: zod_1.z.string().optional().describe('Connector ID'),
+    /**
+     * Destination-specific configuration
+     */
+    config: zod_1.z.record(zod_1.z.any()).describe('Destination configuration'),
+    /**
+     * Write mode
+     */
+    writeMode: zod_1.z.enum([
+        'append', // Add new records
+        'overwrite', // Replace all data
+        'upsert', // Insert or update based on key
+        'merge', // Smart merge based on business rules
+    ]).default('append').describe('How to write data'),
+    /**
+     * Primary key fields for upsert/merge
+     */
+    primaryKey: zod_1.z.array(zod_1.z.string()).optional().describe('Primary key fields'),
+});
+/**
+ * ETL Transformation Type
+ */
+exports.ETLTransformationTypeSchema = zod_1.z.enum([
+    'map', // Field mapping/renaming
+    'filter', // Row filtering
+    'aggregate', // Aggregation/grouping
+    'join', // Joining with other data
+    'script', // Custom JavaScript/Python script
+    'lookup', // Enrich with lookup data
+    'split', // Split one record into multiple
+    'merge', // Merge multiple records into one
+    'normalize', // Data normalization
+    'deduplicate', // Remove duplicates
+]);
+/**
+ * ETL Transformation Configuration
+ */
+exports.ETLTransformationSchema = zod_1.z.object({
+    /**
+     * Transformation name
+     */
+    name: zod_1.z.string().optional().describe('Transformation name'),
+    /**
+     * Transformation type
+     */
+    type: exports.ETLTransformationTypeSchema.describe('Transformation type'),
+    /**
+     * Transformation-specific configuration
+     *
+     * @example For map: { oldField: 'newField' }
+     * @example For filter: { condition: 'status == "active"' }
+     * @example For script: { language: 'javascript', code: '...' }
+     */
+    config: zod_1.z.record(zod_1.z.any()).describe('Transformation config'),
+    /**
+     * Whether to continue on error
+     */
+    continueOnError: zod_1.z.boolean().default(false).describe('Continue on error'),
+});
+/**
+ * ETL Sync Mode
+ */
+exports.ETLSyncModeSchema = zod_1.z.enum([
+    'full', // Full refresh - extract all data every time
+    'incremental', // Only extract changed data
+    'cdc', // Change Data Capture - real-time streaming
+]);
+/**
+ * ETL Pipeline Schema
+ *
+ * Complete definition of a data pipeline from source to destination with transformations.
+ */
+exports.ETLPipelineSchema = zod_1.z.object({
+    /**
+     * Pipeline identifier (snake_case)
+     */
+    name: zod_1.z.string()
+        .regex(/^[a-z_][a-z0-9_]*$/)
+        .describe('Pipeline identifier (snake_case)'),
+    /**
+     * Human-readable pipeline name
+     */
+    label: zod_1.z.string().optional().describe('Pipeline display name'),
+    /**
+     * Pipeline description
+     */
+    description: zod_1.z.string().optional().describe('Pipeline description'),
+    /**
+     * Data source configuration
+     */
+    source: exports.ETLSourceSchema.describe('Data source'),
+    /**
+     * Data destination configuration
+     */
+    destination: exports.ETLDestinationSchema.describe('Data destination'),
+    /**
+     * Transformation steps
+     * Applied in order from source to destination
+     */
+    transformations: zod_1.z.array(exports.ETLTransformationSchema)
+        .optional()
+        .describe('Transformation pipeline'),
+    /**
+     * Sync mode
+     */
+    syncMode: exports.ETLSyncModeSchema.default('full').describe('Sync mode'),
+    /**
+     * Execution schedule (cron expression)
+     *
+     * @example "0 2 * * *" - Daily at 2 AM
+     * @example "0 *\/4 * * *" - Every 4 hours
+     * @example "0 0 * * 0" - Weekly on Sunday
+     */
+    schedule: zod_1.z.string().optional().describe('Cron schedule expression'),
+    /**
+     * Whether pipeline is enabled
+     */
+    enabled: zod_1.z.boolean().default(true).describe('Pipeline enabled status'),
+    /**
+     * Retry configuration for failed runs
+     */
+    retry: zod_1.z.object({
+        maxAttempts: zod_1.z.number().int().min(0).default(3).describe('Max retry attempts'),
+        backoffMs: zod_1.z.number().int().min(0).default(60000).describe('Backoff in milliseconds'),
+    }).optional().describe('Retry configuration'),
+    /**
+     * Notification configuration
+     */
+    notifications: zod_1.z.object({
+        onSuccess: zod_1.z.array(zod_1.z.string()).optional().describe('Email addresses for success notifications'),
+        onFailure: zod_1.z.array(zod_1.z.string()).optional().describe('Email addresses for failure notifications'),
+    }).optional().describe('Notification settings'),
+    /**
+     * Pipeline tags for organization
+     */
+    tags: zod_1.z.array(zod_1.z.string()).optional().describe('Pipeline tags'),
+    /**
+     * Custom metadata
+     */
+    metadata: zod_1.z.record(zod_1.z.any()).optional().describe('Custom metadata'),
+});
+/**
+ * ETL Run Status
+ */
+exports.ETLRunStatusSchema = zod_1.z.enum([
+    'pending', // Queued for execution
+    'running', // Currently executing
+    'succeeded', // Completed successfully
+    'failed', // Failed with errors
+    'cancelled', // Manually cancelled
+    'timeout', // Timed out
+]);
+/**
+ * ETL Pipeline Run Result
+ *
+ * Result of a pipeline execution
+ */
+exports.ETLPipelineRunSchema = zod_1.z.object({
+    /**
+     * Run ID
+     */
+    id: zod_1.z.string().describe('Run identifier'),
+    /**
+     * Pipeline name
+     */
+    pipelineName: zod_1.z.string().describe('Pipeline name'),
+    /**
+     * Run status
+     */
+    status: exports.ETLRunStatusSchema.describe('Run status'),
+    /**
+     * Start timestamp
+     */
+    startedAt: zod_1.z.string().datetime().describe('Start time'),
+    /**
+     * End timestamp
+     */
+    completedAt: zod_1.z.string().datetime().optional().describe('Completion time'),
+    /**
+     * Duration in milliseconds
+     */
+    durationMs: zod_1.z.number().optional().describe('Duration in ms'),
+    /**
+     * Statistics
+     */
+    stats: zod_1.z.object({
+        recordsRead: zod_1.z.number().int().default(0).describe('Records extracted'),
+        recordsWritten: zod_1.z.number().int().default(0).describe('Records loaded'),
+        recordsErrored: zod_1.z.number().int().default(0).describe('Records with errors'),
+        bytesProcessed: zod_1.z.number().int().default(0).describe('Bytes processed'),
+    }).optional().describe('Run statistics'),
+    /**
+     * Error information
+     */
+    error: zod_1.z.object({
+        message: zod_1.z.string().describe('Error message'),
+        code: zod_1.z.string().optional().describe('Error code'),
+        details: zod_1.z.any().optional().describe('Error details'),
+    }).optional().describe('Error information'),
+    /**
+     * Execution logs
+     */
+    logs: zod_1.z.array(zod_1.z.string()).optional().describe('Execution logs'),
+});
+/**
+ * Helper factory for creating ETL pipelines
+ */
+exports.ETL = {
+    /**
+     * Create a simple database-to-database pipeline
+     */
+    databaseSync: (params) => ({
+        name: params.name,
+        source: {
+            type: 'database',
+            config: { table: params.sourceTable },
+        },
+        destination: {
+            type: 'database',
+            config: { table: params.destTable },
+            writeMode: 'upsert',
+        },
+        syncMode: 'incremental',
+        schedule: params.schedule,
+        enabled: true,
+    }),
+    /**
+     * Create an API to database pipeline
+     */
+    apiToDatabase: (params) => ({
+        name: params.name,
+        source: {
+            type: 'api',
+            connector: params.apiConnector,
+            config: {},
+        },
+        destination: {
+            type: 'database',
+            config: { table: params.destTable },
+            writeMode: 'append',
+        },
+        syncMode: 'full',
+        schedule: params.schedule,
+        enabled: true,
+    }),
+};

package/dist/automation/flow.zod.d.ts

@@ -48,20 +48,20 @@ export declare const FlowNodeSchema: z.ZodObject<{
     type: "loop" | "start" | "end" | "script" | "create_record" | "decision" | "assignment" | "update_record" | "delete_record" | "get_record" | "http_request" | "wait" | "subflow";
     label: string;
     id: string;
-    config?: Record<string, any> | undefined;
     position?: {
         x: number;
         y: number;
     } | undefined;
+    config?: Record<string, any> | undefined;
 }, {
     type: "loop" | "start" | "end" | "script" | "create_record" | "decision" | "assignment" | "update_record" | "delete_record" | "get_record" | "http_request" | "wait" | "subflow";
     label: string;
     id: string;
-    config?: Record<string, any> | undefined;
     position?: {
         x: number;
         y: number;
     } | undefined;
+    config?: Record<string, any> | undefined;
 }>;
 /**
  * Flow Edge Schema
@@ -144,20 +144,20 @@ export declare const FlowSchema: z.ZodObject<{
         type: "loop" | "start" | "end" | "script" | "create_record" | "decision" | "assignment" | "update_record" | "delete_record" | "get_record" | "http_request" | "wait" | "subflow";
         label: string;
         id: string;
-        config?: Record<string, any> | undefined;
         position?: {
             x: number;
             y: number;
         } | undefined;
+        config?: Record<string, any> | undefined;
     }, {
         type: "loop" | "start" | "end" | "script" | "create_record" | "decision" | "assignment" | "update_record" | "delete_record" | "get_record" | "http_request" | "wait" | "subflow";
         label: string;
         id: string;
-        config?: Record<string, any> | undefined;
         position?: {
             x: number;
             y: number;
         } | undefined;
+        config?: Record<string, any> | undefined;
     }>, "many">;
     edges: z.ZodArray<z.ZodObject<{
         id: z.ZodString;
@@ -197,11 +197,11 @@ export declare const FlowSchema: z.ZodObject<{
         type: "loop" | "start" | "end" | "script" | "create_record" | "decision" | "assignment" | "update_record" | "delete_record" | "get_record" | "http_request" | "wait" | "subflow";
         label: string;
         id: string;
-        config?: Record<string, any> | undefined;
         position?: {
             x: number;
             y: number;
         } | undefined;
+        config?: Record<string, any> | undefined;
     }[];
     edges: {
         type: "default" | "fault";
@@ -227,11 +227,11 @@ export declare const FlowSchema: z.ZodObject<{
         type: "loop" | "start" | "end" | "script" | "create_record" | "decision" | "assignment" | "update_record" | "delete_record" | "get_record" | "http_request" | "wait" | "subflow";
         label: string;
         id: string;
-        config?: Record<string, any> | undefined;
         position?: {
             x: number;
             y: number;
         } | undefined;
+        config?: Record<string, any> | undefined;
     }[];
     edges: {
         id: string;

package/dist/automation/index.d.ts

@@ -1,4 +1,8 @@
 export * from './workflow.zod';
 export * from './flow.zod';
 export * from './webhook.zod';
+export * from './approval.zod';
+export * from './etl.zod';
+export * from './connector.zod';
+export * from './sync.zod';
 //# sourceMappingURL=index.d.ts.map

package/dist/automation/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/automation/index.ts"],"names":[],"mappings":"AACA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/automation/index.ts"],"names":[],"mappings":"AACA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,WAAW,CAAC;AAC1B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC"}

package/dist/automation/index.js

@@ -17,3 +17,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./workflow.zod"), exports);
 __exportStar(require("./flow.zod"), exports);
 __exportStar(require("./webhook.zod"), exports);
+__exportStar(require("./approval.zod"), exports);
+__exportStar(require("./etl.zod"), exports);
+__exportStar(require("./connector.zod"), exports);
+__exportStar(require("./sync.zod"), exports);