@oronts/vendure-data-hub-plugin 0.1.4 → 0.1.6

package/docs/deployment/troubleshooting.md

@@ -392,15 +392,18 @@ DataHubPlugin.init({
    - Valid JSON body
    - Required headers present
 
-4. **Check webhook logs:**
+4. **Check pipeline run logs for webhook errors:**
    ```graphql
    query {
-     dataHubWebhookLogs(options: { take: 10 }) {
+     dataHubLogs(options: { take: 10 }) {
        items {
-         path
-         statusCode
-         errorMessage
+         id
+         level
+         message
+         stepKey
+         createdAt
        }
+       totalItems
      }
    }
    ```
@@ -530,15 +533,18 @@ DataHubPlugin.init({
    - Check plugin is loaded
    - Verify event handlers registered
 
-4. **Check event logs:**
+4. **Check pipeline run logs for event trigger errors:**
    ```graphql
    query {
-     dataHubEventLogs(options: { take: 10 }) {
+     dataHubLogs(options: { take: 10 }) {
        items {
-         eventType
-         handled
-         errorMessage
+         id
+         level
+         message
+         stepKey
+         createdAt
        }
+       totalItems
      }
    }
    ```
@@ -774,15 +780,18 @@ DataHubPlugin.init({
    - Verify edge connects to gate
    - Check gate isn't skipped by route
 
-3. **Check gate logs:**
+3. **Check pipeline run logs for gate activity:**
    ```graphql
    query {
-     dataHubGateApprovals(options: { take: 10 }) {
+     dataHubLogs(options: { take: 10 }) {
        items {
-         pipelineId
-         status
-         approvedAt
+         id
+         level
+         message
+         stepKey
+         createdAt
        }
+       totalItems
      }
    }
    ```
@@ -962,7 +971,7 @@ DataHubPlugin.init({
 
 ```graphql
 mutation {
-  runDataHubPipeline(id: "pipeline-id", dryRun: true) {
+  startDataHubPipelineRun(pipelineId: "pipeline-id") {
     id
     status
   }
@@ -1059,12 +1068,12 @@ WHERE pipeline_id = 'pipeline-id';
 ```sql
 -- View queue
 SELECT * FROM job_queue
-WHERE queue_name = 'data-hub-run'
+WHERE queue_name = 'data-hub.run'
 AND state = 'PENDING';
 
 -- Clear stuck jobs (use with caution)
 DELETE FROM job_queue
-WHERE queue_name = 'data-hub-run'
+WHERE queue_name = 'data-hub.run'
 AND state = 'PENDING'
 AND created_at < NOW() - INTERVAL '1 hour';
 ```

package/docs/developer-guide/architecture.md

@@ -147,7 +147,7 @@ Runs are processed via Vendure's job queue:
 
 ```typescript
 await jobQueue.add({
-    type: 'data-hub-run',
+    type: 'data-hub.run',
     pipelineId: pipeline.id,
     triggeredBy: 'manual',
 });
@@ -301,7 +301,7 @@ dataHubPipelines() { ... }
 
 @Allow(RunDataHubPipelinePermission)
 @Mutation()
-runDataHubPipeline() { ... }
+startDataHubPipelineRun() { ... }
 ```
 
 ### Secret Encryption

package/docs/developer-guide/extending/README.md

@@ -1161,9 +1161,9 @@ const s3ExportSchema: StepConfigSchema = {
           placeholder: 'exports/${date}/${pipeline}.json' },
         { key: 'format', type: 'select', label: 'Format', required: true,
           options: [
-            { value: 'json', label: 'JSON' },
-            { value: 'ndjson', label: 'NDJSON (Line-delimited)' },
-            { value: 'csv', label: 'CSV' },
+            { value: 'JSON', label: 'JSON' },
+            { value: 'NDJSON', label: 'NDJSON (Line-delimited)' },
+            { value: 'CSV', label: 'CSV' },
           ] },
         { key: 'compress', type: 'boolean', label: 'Compress (gzip)' },
     ],
@@ -1209,15 +1209,15 @@ export const s3Exporter: ExporterAdapter<S3ExportConfig> = {
         let contentType: string;
 
         switch (config.format) {
-            case 'json':
+            case 'JSON':
                 content = JSON.stringify(records, null, 2);
                 contentType = 'application/json';
                 break;
-            case 'ndjson':
+            case 'NDJSON':
                 content = records.map(r => JSON.stringify(r)).join('\n');
                 contentType = 'application/x-ndjson';
                 break;
-            case 'csv':
+            case 'CSV':
                 content = convertToCSV(records);
                 contentType = 'text/csv';
                 break;

package/docs/developer-guide/extending/events.md

@@ -68,8 +68,8 @@ There are two ways to subscribe:
 | Event | Payload | Description |
 |-------|---------|-------------|
 | `TriggerFired` | `{ pipelineCode, triggerType, triggerId }` | Trigger activated a pipeline run |
-| `ScheduleActivated` | `{ pipelineCode, cronExpression, triggerId }` | Cron schedule activated |
-| `ScheduleDeactivated` | `{ pipelineCode, triggerId }` | Cron schedule deactivated |
+| `ScheduleActivated` | `{ pipelineId, pipelineCode, scheduleCount, timestamp }` | Cron schedule activated |
+| `ScheduleDeactivated` | `{ pipelineId, pipelineCode, timestamp }` | Cron schedule deactivated |
 
 ### Webhook Delivery
 

package/docs/developer-guide/graphql-api.md

@@ -1,5 +1,7 @@
 # GraphQL API
 
+> **Important:** Use GraphQL introspection (`{ __schema { ... } }`) or the Vendure Admin UI's API playground for the definitive, up-to-date API reference. The examples below show common usage patterns but field names and signatures may differ from the current implementation.
+
 The Data Hub plugin extends the Vendure Admin API with queries and mutations for pipeline management.
 
 ## Queries
@@ -15,7 +17,6 @@ query {
             id
             code
             name
-            description
             enabled
             createdAt
             updatedAt
@@ -35,7 +36,6 @@ query GetPipeline($id: ID!) {
         id
         code
         name
-        description
         enabled
         definition
         createdAt
@@ -54,7 +54,6 @@ query {
         items {
             id
             code
-            name
             type
             createdAt
         }
@@ -99,7 +98,7 @@ query {
                 type
                 required
                 label
-                default
+                defaultValue
                 options { value label }
             }
         }
@@ -118,9 +117,7 @@ query GetRuns($pipelineId: ID!) {
             id
             status
             startedAt
-            completedAt
-            recordsProcessed
-            recordsFailed
+            finishedAt
             triggeredBy
         }
         totalItems
@@ -139,16 +136,8 @@ query GetRun($id: ID!) {
         status
         startedAt
         completedAt
-        recordsProcessed
-        recordsFailed
+        metrics
         triggeredBy
-        stepMetrics {
-            stepKey
-            recordsIn
-            recordsOut
-            recordsFailed
-            durationMs
-        }
     }
 }
 ```
@@ -158,14 +147,14 @@ query GetRun($id: ID!) {
 Query execution logs:
 
 ```graphql
-query GetLogs($runId: ID!) {
-    dataHubLogs(runId: $runId, options: { take: 100 }) {
+query GetLogs {
+    dataHubLogs(options: { take: 100 }) {
         items {
             id
             level
             message
             stepKey
-            timestamp
+            createdAt
             metadata
         }
         totalItems
@@ -173,24 +162,19 @@ query GetLogs($runId: ID!) {
 }
 ```
 
-### dataHubRecordErrors
+### dataHubRunErrors
 
-Query failed records:
+Query failed records for a specific run:
 
 ```graphql
-query {
-    dataHubRecordErrors(options: { take: 50 }) {
-        items {
-            id
-            pipelineId
-            runId
-            stepKey
-            recordData
-            errorMessage
-            createdAt
-            retryCount
-        }
-        totalItems
+query RunErrors($runId: ID!) {
+    dataHubRunErrors(runId: $runId) {
+        id
+        stepKey
+        message
+        payload
+        stackTrace
+        createdAt
     }
 }
 ```
@@ -232,7 +216,6 @@ Variables:
     "input": {
         "code": "my-pipeline",
         "name": "My Pipeline",
-        "description": "Pipeline description",
         "definition": {
             "version": 1,
             "steps": [],
@@ -280,13 +263,13 @@ mutation DeletePipeline($id: ID!) {
 }
 ```
 
-### runDataHubPipeline
+### startDataHubPipelineRun
 
 Execute a pipeline:
 
 ```graphql
-mutation RunPipeline($id: ID!, $parameters: JSON) {
-    runDataHubPipeline(id: $id, parameters: $parameters) {
+mutation RunPipeline($pipelineId: ID!) {
+    startDataHubPipelineRun(pipelineId: $pipelineId) {
         id
         status
     }
@@ -298,10 +281,10 @@ mutation RunPipeline($id: ID!, $parameters: JSON) {
 Cancel a running pipeline:
 
 ```graphql
-mutation CancelRun($runId: ID!) {
-    cancelDataHubPipelineRun(runId: $runId) {
-        success
-        message
+mutation CancelRun($id: ID!) {
+    cancelDataHubPipelineRun(id: $id) {
+        id
+        status
     }
 }
 ```
@@ -311,14 +294,13 @@ mutation CancelRun($runId: ID!) {
 Validate a pipeline definition:
 
 ```graphql
-mutation Validate($definition: JSON!) {
+query Validate($definition: JSON!) {
     validateDataHubPipelineDefinition(definition: $definition) {
         isValid
-        errors
         issues {
             stepKey
             message
-            severity
+            reason
         }
     }
 }
@@ -363,20 +345,6 @@ mutation DeleteConnection($id: ID!) {
 }
 ```
 
-### testDataHubConnection
-
-Test a connection:
-
-```graphql
-mutation TestConnection($id: ID!) {
-    testDataHubConnection(id: $id) {
-        success
-        message
-        latencyMs
-    }
-}
-```
-
 ### createDataHubSecret
 
 Create a secret:
@@ -427,78 +395,29 @@ mutation DeleteSecret($id: ID!) {
 }
 ```
 
-### retryDataHubRecordErrors
+### retryDataHubRecord
 
-Retry failed records:
+Retry a single failed record with optional JSON patch:
 
 ```graphql
-mutation RetryErrors($ids: [ID!]!) {
-    retryDataHubRecordErrors(ids: $ids) {
-        success
-        retriedCount
-        failedCount
-    }
-}
-```
-
-### deleteDataHubRecordErrors
-
-Delete error records:
-
-```graphql
-mutation DeleteErrors($ids: [ID!]!) {
-    deleteDataHubRecordErrors(ids: $ids) {
-        success
-        deletedCount
-    }
+mutation RetryRecord($errorId: ID!, $patch: JSON) {
+    retryDataHubRecord(errorId: $errorId, patch: $patch)
 }
 ```
 
-### setDataHubSettings
+### updateDataHubSettings
 
 Update plugin settings:
 
 ```graphql
 mutation UpdateSettings($input: DataHubSettingsInput!) {
-    setDataHubSettings(input: $input) {
+    updateDataHubSettings(input: $input) {
         retentionDaysRuns
         retentionDaysErrors
     }
 }
 ```
 
-## Subscriptions
-
-### dataHubPipelineRunUpdated
-
-Subscribe to run updates:
-
-```graphql
-subscription OnRunUpdate($pipelineId: ID) {
-    dataHubPipelineRunUpdated(pipelineId: $pipelineId) {
-        id
-        status
-        recordsProcessed
-        recordsFailed
-    }
-}
-```
-
-### dataHubLogAdded
-
-Subscribe to new logs:
-
-```graphql
-subscription OnLog($runId: ID!) {
-    dataHubLogAdded(runId: $runId) {
-        id
-        level
-        message
-        timestamp
-    }
-}
-```
-
 ## TypeScript Client
 
 Using with `@vendure/admin-ui-plugin` or custom clients:
@@ -507,8 +426,8 @@ Using with `@vendure/admin-ui-plugin` or custom clients:
 import { gql } from 'graphql-tag';
 
 const RUN_PIPELINE = gql`
-    mutation RunPipeline($id: ID!) {
-        runDataHubPipeline(id: $id) {
+    mutation RunPipeline($pipelineId: ID!) {
+        startDataHubPipelineRun(pipelineId: $pipelineId) {
             id
             status
         }
@@ -518,7 +437,7 @@ const RUN_PIPELINE = gql`
 // Execute
 const result = await adminClient.mutate({
     mutation: RUN_PIPELINE,
-    variables: { id: '1' },
+    variables: { pipelineId: '1' },
 });
 ```
 

package/docs/examples/validation-error-messages.md

@@ -5,7 +5,7 @@ The `ValidationBuilder` now provides rich, contextual error messages that help d
 ## Basic Usage with Context
 
 ```typescript
-import { ValidationBuilder } from '@vendure-plugins/data-hub';
+import { ValidationBuilder } from '@oronts/vendure-data-hub-plugin';
 
 // Simple validation with record identifier
 const result = new ValidationBuilder()

package/docs/getting-started/quick-start.md

@@ -130,7 +130,7 @@ export const config: VendureConfig = {
 
 ```graphql
 mutation {
-    runDataHubPipeline(id: "1") {
+    startDataHubPipelineRun(pipelineId: "1") {
         id
         status
     }

package/docs/guides/multi-entity.md

@@ -592,6 +592,5 @@ The default modes are chosen to be safe and match most use cases:
 ## Related Documentation
 
 - [Loaders Reference](../reference/loaders.md) - Complete API reference
-- [Production Standards](../../PIPELINE-PRODUCTION-STANDARDS.md) - Best practices
 - [Multi-Language Guide](./multi-language.md) - Translation handling
 - [Multi-Channel Guide](./multi-channel.md) - Channel assignment

package/docs/reference/extractors.md

@@ -170,18 +170,19 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
 
 | Field | Type | Required | Description |
 |-------|------|----------|-------------|
-| `url` | string | Yes | GraphQL endpoint URL |
+| `endpoint` | string | Yes | GraphQL endpoint URL |
 | `query` | string | Yes | GraphQL query |
 | `connectionCode` | string | No | HTTP connection to use (optional) |
 | `headers` | json | No | Request headers (JSON object) |
 | `variables` | json | No | Query variables (JSON object) |
-| `dataPath` | string | No | JSON path to items in response |
-| `pagination.type` | select | No | Pagination type: NONE, OFFSET, CURSOR, RELAY |
-| `pagination.limit` | number | No | Page size |
-| `pagination.maxPages` | number | No | Maximum pages to fetch |
-| `pagination.cursorPath` | string | No | JSON path to cursor in response |
-| `pagination.cursorVariable` | string | No | Variable name for cursor |
-| `pagination.hasNextPagePath` | string | No | JSON path to hasNextPage boolean |
+| `itemsField` | string | No | Field name containing items in response |
+| `edgesField` | string | No | Field name for Relay-style edges |
+| `nodeField` | string | No | Field name for node within each edge |
+| `cursorVar` | string | No | Variable name for cursor pagination |
+| `nextCursorField` | string | No | Field name for next cursor in response |
+| `pageInfoField` | string | No | Field name for pageInfo object |
+| `hasNextPageField` | string | No | Field name for hasNextPage boolean |
+| `endCursorField` | string | No | Field name for endCursor in pageInfo |
 | `timeoutMs` | number | No | Request timeout in milliseconds |
 
 ### Example - Basic Query
@@ -189,7 +190,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
 ```typescript
 .extract('query-graphql', {
     adapterCode: 'graphql',
-    url: 'https://api.example.com/graphql',
+    endpoint: 'https://api.example.com/graphql',
     query: `
         query GetProducts($limit: Int) {
             products(limit: $limit) {
@@ -200,7 +201,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
         }
     `,
     variables: { limit: 100 },
-    dataPath: 'data.products',
+    itemsField: 'products',
 })
 ```
 
@@ -209,7 +210,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
 ```typescript
 .extract('query-with-offset', {
     adapterCode: 'graphql',
-    url: 'https://api.example.com/graphql',
+    endpoint: 'https://api.example.com/graphql',
     query: `
         query GetProducts($skip: Int, $take: Int) {
             products(skip: $skip, take: $take) {
@@ -218,11 +219,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
             }
         }
     `,
-    dataPath: 'data.products.items',
-    pagination: {
-        type: 'OFFSET',
-        limit: 100,
-    },
+    itemsField: 'products.items',
 })
 ```
 
@@ -231,7 +228,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
 ```typescript
 .extract('query-with-cursor', {
     adapterCode: 'graphql',
-    url: 'https://api.example.com/graphql',
+    endpoint: 'https://api.example.com/graphql',
     query: `
         query GetProducts($cursor: String) {
             products(first: 100, after: $cursor) {
@@ -248,13 +245,11 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
             }
         }
     `,
-    dataPath: 'data.products.edges',
-    pagination: {
-        type: 'RELAY',
-        cursorPath: 'data.products.pageInfo.endCursor',
-        hasNextPagePath: 'data.products.pageInfo.hasNextPage',
-        cursorVariable: 'cursor',
-    },
+    edgesField: 'products.edges',
+    pageInfoField: 'products.pageInfo',
+    hasNextPageField: 'hasNextPage',
+    endCursorField: 'endCursor',
+    cursorVar: 'cursor',
 })
 ```
 

package/docs/reference/operators-complete.md

@@ -1248,4 +1248,4 @@ Execute inline JavaScript code to transform records. Use for complex logic that
 
 ## Custom Operators
 
-You can register custom operators using the SDK. See [Custom Operators Guide](../guides/custom-operators.md) for details.
+You can register custom operators using the SDK. See [Custom Operators Guide](../developer-guide/extending/custom-operators.md) for details.

package/docs/user-guide/external-integrations.md

@@ -12,7 +12,7 @@ Index your data to search engines with automatic circuit breaker protection.
 |--------|--------------|----------|
 | MeiliSearch | `meilisearch` | Full-text search, faceting |
 | Elasticsearch | `elasticsearch` | Advanced querying, analytics |
-| OpenSearch | `elasticsearch` | AWS-compatible (uses Elasticsearch adapter) |
+| OpenSearch | `opensearch` | AWS-compatible, Elasticsearch-compatible API |
 | Algolia | `algolia` | Instant search, relevance tuning |
 | Typesense | `typesense` | Typo-tolerant, fast |
 

package/docs/user-guide/queue-messaging.md

@@ -356,7 +356,7 @@ const productChangeFanout = createPipeline()
     .name('product-change-fanout')
     .trigger('product-event', {
         type: 'EVENT',
-        eventType: 'ProductEvent',
+        event: 'ProductEvent',
     })
     .extract('from-event', { adapterCode: 'inMemory' })
     .sink('to-search-queue', {
@@ -463,16 +463,19 @@ Check queue consumer status:
 
 ```graphql
 query {
-    dataHubQueueStatus(connectionCode: "rabbitmq-main") {
-        connected
-        consumers {
-            pipelineCode
-            queue
-            messagesProcessed
-            messagesPerMinute
-            lastMessageAt
-            errors
-        }
+    dataHubQueueStats {
+        pending
+        running
+        failed
+        completedToday
+    }
+    dataHubConsumers {
+        pipelineCode
+        queueName
+        isActive
+        messagesProcessed
+        messagesFailed
+        lastMessageAt
     }
 }
 ```

package/docs/user-guide/recipes.md

@@ -1689,5 +1689,4 @@ const eventDrivenSync = createPipeline()
 
 - [Pipeline Builder Guide](../developer-guide/dsl/pipeline-builder.md) - DSL API reference
 - [Operators Reference](../developer-guide/dsl/operators.md) - All operators
-- [Templates Guide](./templates.md) - Pre-built pipeline templates
 - [Performance Tuning](../deployment/performance.md) - Optimization techniques

package/docs/user-guide/wizards.md

@@ -761,7 +761,6 @@ Before importing, preview what will happen:
 
 ## See Also
 
-- [Templates Guide](./templates.md) - Pre-built templates
 - [Operators Guide](../developer-guide/dsl/operators.md) - Field transformations
 - [Performance Guide](../deployment/performance.md) - Optimization
 - [Troubleshooting Guide](../deployment/troubleshooting.md) - Common issues

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@oronts/vendure-data-hub-plugin",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "ETL data integration plugin for Vendure e-commerce",
   "author": {
     "name": "Oronts Website",
@@ -19,6 +19,7 @@
   "files": [
     "dist",
     "shared",
+    "sdk",
     "dashboard",
     "docs",
     "README.md",
@@ -85,6 +86,7 @@
     "glob": "^10.0.0",
     "ioredis": "^5.9.1",
     "js-yaml": "^4.1.0",
+    "minimatch": "^9.0.0",
     "multer": "^2.0.0",
     "mysql2": "^3.11.0",
     "nodemailer": "^7.0.12",