@oronts/vendure-data-hub-plugin 0.1.5 → 0.1.6

package/docs/developer-guide/graphql-api.md

@@ -1,5 +1,7 @@
 # GraphQL API
 
+> **Important:** Use GraphQL introspection (`{ __schema { ... } }`) or the Vendure Admin UI's API playground for the definitive, up-to-date API reference. The examples below show common usage patterns but field names and signatures may differ from the current implementation.
+
 The Data Hub plugin extends the Vendure Admin API with queries and mutations for pipeline management.
 
 ## Queries
@@ -15,7 +17,6 @@ query {
             id
             code
             name
-            description
             enabled
             createdAt
             updatedAt
@@ -35,7 +36,6 @@ query GetPipeline($id: ID!) {
         id
         code
         name
-        description
         enabled
         definition
         createdAt
@@ -54,7 +54,6 @@ query {
         items {
             id
             code
-            name
             type
             createdAt
         }
@@ -99,7 +98,7 @@ query {
                 type
                 required
                 label
-                default
+                defaultValue
                 options { value label }
             }
         }
@@ -118,9 +117,7 @@ query GetRuns($pipelineId: ID!) {
             id
             status
             startedAt
-            completedAt
-            recordsProcessed
-            recordsFailed
+            finishedAt
             triggeredBy
         }
         totalItems
@@ -139,16 +136,8 @@ query GetRun($id: ID!) {
         status
         startedAt
         completedAt
-        recordsProcessed
-        recordsFailed
+        metrics
         triggeredBy
-        stepMetrics {
-            stepKey
-            recordsIn
-            recordsOut
-            recordsFailed
-            durationMs
-        }
     }
 }
 ```
@@ -158,14 +147,14 @@ query GetRun($id: ID!) {
 Query execution logs:
 
 ```graphql
-query GetLogs($runId: ID!) {
-    dataHubLogs(runId: $runId, options: { take: 100 }) {
+query GetLogs {
+    dataHubLogs(options: { take: 100 }) {
         items {
             id
             level
             message
             stepKey
-            timestamp
+            createdAt
             metadata
         }
         totalItems
@@ -173,24 +162,19 @@ query GetLogs($runId: ID!) {
 }
 ```
 
-### dataHubRecordErrors
+### dataHubRunErrors
 
-Query failed records:
+Query failed records for a specific run:
 
 ```graphql
-query {
-    dataHubRecordErrors(options: { take: 50 }) {
-        items {
-            id
-            pipelineId
-            runId
-            stepKey
-            recordData
-            errorMessage
-            createdAt
-            retryCount
-        }
-        totalItems
+query RunErrors($runId: ID!) {
+    dataHubRunErrors(runId: $runId) {
+        id
+        stepKey
+        message
+        payload
+        stackTrace
+        createdAt
     }
 }
 ```
@@ -232,7 +216,6 @@ Variables:
     "input": {
         "code": "my-pipeline",
         "name": "My Pipeline",
-        "description": "Pipeline description",
         "definition": {
             "version": 1,
             "steps": [],
@@ -280,13 +263,13 @@ mutation DeletePipeline($id: ID!) {
 }
 ```
 
-### runDataHubPipeline
+### startDataHubPipelineRun
 
 Execute a pipeline:
 
 ```graphql
-mutation RunPipeline($id: ID!, $parameters: JSON) {
-    runDataHubPipeline(id: $id, parameters: $parameters) {
+mutation RunPipeline($pipelineId: ID!) {
+    startDataHubPipelineRun(pipelineId: $pipelineId) {
         id
         status
     }
@@ -298,10 +281,10 @@ mutation RunPipeline($id: ID!, $parameters: JSON) {
 Cancel a running pipeline:
 
 ```graphql
-mutation CancelRun($runId: ID!) {
-    cancelDataHubPipelineRun(runId: $runId) {
-        success
-        message
+mutation CancelRun($id: ID!) {
+    cancelDataHubPipelineRun(id: $id) {
+        id
+        status
     }
 }
 ```
@@ -311,14 +294,13 @@ mutation CancelRun($runId: ID!) {
 Validate a pipeline definition:
 
 ```graphql
-mutation Validate($definition: JSON!) {
+query Validate($definition: JSON!) {
     validateDataHubPipelineDefinition(definition: $definition) {
         isValid
-        errors
         issues {
             stepKey
             message
-            severity
+            reason
         }
     }
 }
@@ -363,20 +345,6 @@ mutation DeleteConnection($id: ID!) {
 }
 ```
 
-### testDataHubConnection
-
-Test a connection:
-
-```graphql
-mutation TestConnection($id: ID!) {
-    testDataHubConnection(id: $id) {
-        success
-        message
-        latencyMs
-    }
-}
-```
-
 ### createDataHubSecret
 
 Create a secret:
@@ -427,78 +395,29 @@ mutation DeleteSecret($id: ID!) {
 }
 ```
 
-### retryDataHubRecordErrors
+### retryDataHubRecord
 
-Retry failed records:
+Retry a single failed record with optional JSON patch:
 
 ```graphql
-mutation RetryErrors($ids: [ID!]!) {
-    retryDataHubRecordErrors(ids: $ids) {
-        success
-        retriedCount
-        failedCount
-    }
-}
-```
-
-### deleteDataHubRecordErrors
-
-Delete error records:
-
-```graphql
-mutation DeleteErrors($ids: [ID!]!) {
-    deleteDataHubRecordErrors(ids: $ids) {
-        success
-        deletedCount
-    }
+mutation RetryRecord($errorId: ID!, $patch: JSON) {
+    retryDataHubRecord(errorId: $errorId, patch: $patch)
 }
 ```
 
-### setDataHubSettings
+### updateDataHubSettings
 
 Update plugin settings:
 
 ```graphql
 mutation UpdateSettings($input: DataHubSettingsInput!) {
-    setDataHubSettings(input: $input) {
+    updateDataHubSettings(input: $input) {
         retentionDaysRuns
         retentionDaysErrors
     }
 }
 ```
 
-## Subscriptions
-
-### dataHubPipelineRunUpdated
-
-Subscribe to run updates:
-
-```graphql
-subscription OnRunUpdate($pipelineId: ID) {
-    dataHubPipelineRunUpdated(pipelineId: $pipelineId) {
-        id
-        status
-        recordsProcessed
-        recordsFailed
-    }
-}
-```
-
-### dataHubLogAdded
-
-Subscribe to new logs:
-
-```graphql
-subscription OnLog($runId: ID!) {
-    dataHubLogAdded(runId: $runId) {
-        id
-        level
-        message
-        timestamp
-    }
-}
-```
-
 ## TypeScript Client
 
 Using with `@vendure/admin-ui-plugin` or custom clients:
@@ -507,8 +426,8 @@ Using with `@vendure/admin-ui-plugin` or custom clients:
 import { gql } from 'graphql-tag';
 
 const RUN_PIPELINE = gql`
-    mutation RunPipeline($id: ID!) {
-        runDataHubPipeline(id: $id) {
+    mutation RunPipeline($pipelineId: ID!) {
+        startDataHubPipelineRun(pipelineId: $pipelineId) {
             id
             status
         }
@@ -518,7 +437,7 @@ const RUN_PIPELINE = gql`
 // Execute
 const result = await adminClient.mutate({
     mutation: RUN_PIPELINE,
-    variables: { id: '1' },
+    variables: { pipelineId: '1' },
 });
 ```
 

package/docs/examples/validation-error-messages.md

@@ -5,7 +5,7 @@ The `ValidationBuilder` now provides rich, contextual error messages that help d
 ## Basic Usage with Context
 
 ```typescript
-import { ValidationBuilder } from '@vendure-plugins/data-hub';
+import { ValidationBuilder } from '@oronts/vendure-data-hub-plugin';
 
 // Simple validation with record identifier
 const result = new ValidationBuilder()

package/docs/getting-started/quick-start.md

@@ -130,7 +130,7 @@ export const config: VendureConfig = {
 
 ```graphql
 mutation {
-    runDataHubPipeline(id: "1") {
+    startDataHubPipelineRun(pipelineId: "1") {
         id
         status
     }

package/docs/guides/multi-entity.md

@@ -592,6 +592,5 @@ The default modes are chosen to be safe and match most use cases:
 ## Related Documentation
 
 - [Loaders Reference](../reference/loaders.md) - Complete API reference
-- [Production Standards](../../PIPELINE-PRODUCTION-STANDARDS.md) - Best practices
 - [Multi-Language Guide](./multi-language.md) - Translation handling
 - [Multi-Channel Guide](./multi-channel.md) - Channel assignment

package/docs/reference/extractors.md

@@ -170,18 +170,19 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
 
 | Field | Type | Required | Description |
 |-------|------|----------|-------------|
-| `url` | string | Yes | GraphQL endpoint URL |
+| `endpoint` | string | Yes | GraphQL endpoint URL |
 | `query` | string | Yes | GraphQL query |
 | `connectionCode` | string | No | HTTP connection to use (optional) |
 | `headers` | json | No | Request headers (JSON object) |
 | `variables` | json | No | Query variables (JSON object) |
-| `dataPath` | string | No | JSON path to items in response |
-| `pagination.type` | select | No | Pagination type: NONE, OFFSET, CURSOR, RELAY |
-| `pagination.limit` | number | No | Page size |
-| `pagination.maxPages` | number | No | Maximum pages to fetch |
-| `pagination.cursorPath` | string | No | JSON path to cursor in response |
-| `pagination.cursorVariable` | string | No | Variable name for cursor |
-| `pagination.hasNextPagePath` | string | No | JSON path to hasNextPage boolean |
+| `itemsField` | string | No | Field name containing items in response |
+| `edgesField` | string | No | Field name for Relay-style edges |
+| `nodeField` | string | No | Field name for node within each edge |
+| `cursorVar` | string | No | Variable name for cursor pagination |
+| `nextCursorField` | string | No | Field name for next cursor in response |
+| `pageInfoField` | string | No | Field name for pageInfo object |
+| `hasNextPageField` | string | No | Field name for hasNextPage boolean |
+| `endCursorField` | string | No | Field name for endCursor in pageInfo |
 | `timeoutMs` | number | No | Request timeout in milliseconds |
 
 ### Example - Basic Query
@@ -189,7 +190,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
 ```typescript
 .extract('query-graphql', {
     adapterCode: 'graphql',
-    url: 'https://api.example.com/graphql',
+    endpoint: 'https://api.example.com/graphql',
     query: `
         query GetProducts($limit: Int) {
             products(limit: $limit) {
@@ -200,7 +201,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
         }
     `,
     variables: { limit: 100 },
-    dataPath: 'data.products',
+    itemsField: 'products',
 })
 ```
 
@@ -209,7 +210,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
 ```typescript
 .extract('query-with-offset', {
     adapterCode: 'graphql',
-    url: 'https://api.example.com/graphql',
+    endpoint: 'https://api.example.com/graphql',
     query: `
         query GetProducts($skip: Int, $take: Int) {
             products(skip: $skip, take: $take) {
@@ -218,11 +219,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
             }
         }
     `,
-    dataPath: 'data.products.items',
-    pagination: {
-        type: 'OFFSET',
-        limit: 100,
-    },
+    itemsField: 'products.items',
 })
 ```
 
@@ -231,7 +228,7 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
 ```typescript
 .extract('query-with-cursor', {
     adapterCode: 'graphql',
-    url: 'https://api.example.com/graphql',
+    endpoint: 'https://api.example.com/graphql',
     query: `
         query GetProducts($cursor: String) {
             products(first: 100, after: $cursor) {
@@ -248,13 +245,11 @@ Query external GraphQL endpoints with cursor/offset/Relay pagination support.
             }
         }
     `,
-    dataPath: 'data.products.edges',
-    pagination: {
-        type: 'RELAY',
-        cursorPath: 'data.products.pageInfo.endCursor',
-        hasNextPagePath: 'data.products.pageInfo.hasNextPage',
-        cursorVariable: 'cursor',
-    },
+    edgesField: 'products.edges',
+    pageInfoField: 'products.pageInfo',
+    hasNextPageField: 'hasNextPage',
+    endCursorField: 'endCursor',
+    cursorVar: 'cursor',
 })
 ```
 

package/docs/reference/operators-complete.md

@@ -1248,4 +1248,4 @@ Execute inline JavaScript code to transform records. Use for complex logic that
 
 ## Custom Operators
 
-You can register custom operators using the SDK. See [Custom Operators Guide](../guides/custom-operators.md) for details.
+You can register custom operators using the SDK. See [Custom Operators Guide](../developer-guide/extending/custom-operators.md) for details.

package/docs/user-guide/external-integrations.md

@@ -12,7 +12,7 @@ Index your data to search engines with automatic circuit breaker protection.
 |--------|--------------|----------|
 | MeiliSearch | `meilisearch` | Full-text search, faceting |
 | Elasticsearch | `elasticsearch` | Advanced querying, analytics |
-| OpenSearch | `elasticsearch` | AWS-compatible (uses Elasticsearch adapter) |
+| OpenSearch | `opensearch` | AWS-compatible, Elasticsearch-compatible API |
 | Algolia | `algolia` | Instant search, relevance tuning |
 | Typesense | `typesense` | Typo-tolerant, fast |
 

package/docs/user-guide/queue-messaging.md

@@ -356,7 +356,7 @@ const productChangeFanout = createPipeline()
     .name('product-change-fanout')
     .trigger('product-event', {
         type: 'EVENT',
-        eventType: 'ProductEvent',
+        event: 'ProductEvent',
     })
     .extract('from-event', { adapterCode: 'inMemory' })
     .sink('to-search-queue', {
@@ -463,16 +463,19 @@ Check queue consumer status:
 
 ```graphql
 query {
-    dataHubQueueStatus(connectionCode: "rabbitmq-main") {
-        connected
-        consumers {
-            pipelineCode
-            queue
-            messagesProcessed
-            messagesPerMinute
-            lastMessageAt
-            errors
-        }
+    dataHubQueueStats {
+        pending
+        running
+        failed
+        completedToday
+    }
+    dataHubConsumers {
+        pipelineCode
+        queueName
+        isActive
+        messagesProcessed
+        messagesFailed
+        lastMessageAt
     }
 }
 ```

package/docs/user-guide/recipes.md

@@ -1689,5 +1689,4 @@ const eventDrivenSync = createPipeline()
 
 - [Pipeline Builder Guide](../developer-guide/dsl/pipeline-builder.md) - DSL API reference
 - [Operators Reference](../developer-guide/dsl/operators.md) - All operators
-- [Templates Guide](./templates.md) - Pre-built pipeline templates
 - [Performance Tuning](../deployment/performance.md) - Optimization techniques

package/docs/user-guide/wizards.md

@@ -761,7 +761,6 @@ Before importing, preview what will happen:
 
 ## See Also
 
-- [Templates Guide](./templates.md) - Pre-built templates
 - [Operators Guide](../developer-guide/dsl/operators.md) - Field transformations
 - [Performance Guide](../deployment/performance.md) - Optimization
 - [Troubleshooting Guide](../deployment/troubleshooting.md) - Common issues

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@oronts/vendure-data-hub-plugin",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "ETL data integration plugin for Vendure e-commerce",
   "author": {
     "name": "Oronts Website",
@@ -19,6 +19,7 @@
   "files": [
     "dist",
     "shared",
+    "sdk",
     "dashboard",
     "docs",
     "README.md",
@@ -85,6 +86,7 @@
     "glob": "^10.0.0",
     "ioredis": "^5.9.1",
     "js-yaml": "^4.1.0",
+    "minimatch": "^9.0.0",
     "multer": "^2.0.0",
     "mysql2": "^3.11.0",
     "nodemailer": "^7.0.12",

package/sdk/types/connection-types.ts

@@ -0,0 +1,9 @@
+/**
+ * Re-export connection types from src for external SDK consumers.
+ *
+ * This is an intentional layer bridge - the top-level sdk/ directory
+ * provides a stable public API surface for SDK consumers, while the actual
+ * implementation lives in src/sdk/. External packages import from here;
+ * internal code imports from src/sdk/ directly.
+ */
+export * from '../../src/sdk/types/connection-types';

package/shared/constants/step-type-configs.ts

@@ -1,5 +1,5 @@
 /**
- * Step type configuration data -- single source of truth.
+ * Step type configuration data. Single source of truth.
  *
  * Both the backend (`src/constants/enum-metadata.ts`) and the dashboard
  * (`dashboard/constants/steps.ts`) derive their step configs from this
@@ -43,7 +43,7 @@ export const SHARED_STEP_TYPE_CONFIGS: readonly SharedStepTypeConfig[] = [
     { type: 'TRIGGER', label: 'Trigger', description: 'Start point for the pipeline', icon: 'Play', color: '#6366f1', bgColor: '#eef2ff', borderColor: '#a5b4fc', inputs: 0, outputs: 1, adapterType: 'TRIGGER', nodeType: 'trigger', defaultConfig: { type: 'MANUAL', enabled: true } },
     { type: 'EXTRACT', label: 'Extract', description: 'Pull data from external sources', icon: 'Download', color: '#3b82f6', bgColor: '#eff6ff', borderColor: '#93c5fd', inputs: 1, outputs: 1, adapterType: 'EXTRACTOR', nodeType: 'source' },
     { type: 'TRANSFORM', label: 'Transform', description: 'Map and modify data fields', icon: 'RefreshCw', color: '#8b5cf6', bgColor: '#f5f3ff', borderColor: '#c4b5fd', inputs: 1, outputs: 1, adapterType: 'OPERATOR', nodeType: 'transform' },
-    { type: 'VALIDATE', label: 'Validate', description: 'Check data against rules', icon: 'CheckCircle', color: '#f59e0b', bgColor: '#fffbeb', borderColor: '#fcd34d', inputs: 1, outputs: 1, adapterType: 'VALIDATOR', nodeType: 'validate', defaultConfig: { mode: 'FAIL_FAST' } },
+    { type: 'VALIDATE', label: 'Validate', description: 'Check data against rules', icon: 'CheckCircle', color: '#f59e0b', bgColor: '#fffbeb', borderColor: '#fcd34d', inputs: 1, outputs: 1, adapterType: 'VALIDATOR', nodeType: 'validate', defaultConfig: { mode: 'STRICT' } },
     { type: 'ENRICH', label: 'Enrich', description: 'Add or enrich data fields', icon: 'Sparkles', color: '#10b981', bgColor: '#ecfdf5', borderColor: '#6ee7b7', inputs: 1, outputs: 1, adapterType: 'ENRICHER', nodeType: 'enrich' },
     { type: 'ROUTE', label: 'Route', description: 'Branch data based on conditions', icon: 'GitBranch', color: '#f97316', bgColor: '#fff7ed', borderColor: '#fdba74', inputs: 1, outputs: 2, adapterType: 'ROUTER', nodeType: 'condition', defaultConfig: { branches: [] } },
     { type: 'LOAD', label: 'Load', description: 'Write data to Vendure', icon: 'Upload', color: '#ef4444', bgColor: '#fef2f2', borderColor: '#fca5a5', inputs: 1, outputs: 0, adapterType: 'LOADER', nodeType: 'load' },

package/shared/types/adapter-config.types.ts

@@ -448,7 +448,7 @@ export type ConditionsMode = 'REPLACE_ALL' | 'MERGE' | 'SKIP';
 export type ActionsMode = 'REPLACE_ALL' | 'MERGE' | 'SKIP';
 
 /** How to handle customer groups */
-export type GroupsMode = 'add' | 'set';
+export type GroupsMode = 'ADD' | 'SET';
 
 /** Customer Upsert Loader */
 export interface CustomerUpsertLoaderConfig {
@@ -1121,7 +1121,7 @@ export interface RouteConfig {
         name: string;
         when: Array<{
             field: string;
-            cmp: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte' | 'contains' | 'startsWith' | 'endsWith' | 'matches' | 'isNull' | 'exists' | 'in';
+            cmp: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte' | 'contains' | 'notContains' | 'startsWith' | 'endsWith' | 'matches' | 'regex' | 'isNull' | 'exists' | 'in' | 'notIn';
             value: unknown;
         }>;
     }>;
@@ -1138,7 +1138,7 @@ export type UpdateCatalogLoaders = 'productUpsert' | 'variantUpsert' | 'stockAdj
 export type UpdateCustomerLoaders = 'customerUpsert' | 'customerGroupUpsert';
 
 /** Loader adapter codes that require UpdateOrder permission */
-export type UpdateOrderLoaders = 'orderNote' | 'orderTransition' | 'applyCoupon';
+export type UpdateOrderLoaders = 'orderUpsert' | 'orderNote' | 'orderTransition' | 'applyCoupon';
 
 /** Loader adapter codes that require UpdatePromotion permission */
 export type UpdatePromotionLoaders = 'promotionUpsert';

package/shared/types/adapter.types.ts

@@ -80,7 +80,7 @@ export type DependsOnOperator = 'eq' | 'ne' | 'in' | 'exists';
 /**
  * Option for select/multiselect fields
  *
- * @see AdapterSchemaField — uses SelectOption[] in its `options` property
+ * @see AdapterSchemaField - uses SelectOption[] in its `options` property
  */
 export interface SelectOption {
     /** Value stored in configuration */

package/shared/types/execution.types.ts

@@ -9,10 +9,7 @@ import { JsonObject, JsonValue } from './json.types';
 import { StepMetrics, StepError } from './step.types';
 
 /**
- * Metrics for a complete pipeline run
- */
-/**
- * Pipeline metrics that can be updated during runtime.
+ * Pipeline metrics collected during runtime.
  * Supports additional custom metric keys beyond the standard ones.
  */
 export interface PipelineMetrics {
@@ -169,7 +166,7 @@ export interface InvalidRecord {
 /**
  * Validation error for a specific field in a record
  */
-interface ValidationErrorRecord {
+export interface ValidationErrorRecord {
     /** Field that failed validation */
     readonly field?: string;
     /** Validation rule that failed */

package/shared/types/extractor.types.ts

@@ -28,7 +28,7 @@ export interface ExtractorConfig {
     /** Authentication settings (can override connection auth) */
     auth?: AuthConfig;
 
-    /** Allow additional properties */
+    /** Extractor-specific properties */
     [key: string]: unknown;
 }
 

package/shared/types/index.ts

@@ -166,7 +166,6 @@ export type {
     WebhookTriggerConfig,
     EventTriggerConfig,
     QueueTypeValue,
-    QueueTriggerConfig,
     FileWatchEvent,
     FileWatchTriggerConfig,
     AckMode,
@@ -186,6 +185,7 @@ export type {
     SinkResult,
     ValidationResult,
     InvalidRecord,
+    ValidationErrorRecord,
 } from './execution.types';
 
 export type {

package/shared/types/pipeline.types.ts

@@ -7,7 +7,7 @@ import { Throughput, PipelineStepDefinition, PipelineEdge, PipelineCapabilities,
 import { FieldMapping } from './mapping.types';
 import { FilterCondition } from './filter.types';
 import { PipelineHooksConfig, PipelineHooks } from './hook.types';
-import { TriggerConfig } from './trigger.types';
+import { TriggerConfig, WebhookAuthType } from './trigger.types';
 
 // PIPELINE TYPE ENUMS
 
@@ -339,7 +339,6 @@ interface FileFormatConfig {
     xlsx?: XlsxFormatOptions;
 }
 
-type WebhookAuthMode = 'NONE' | 'API_KEY' | 'HMAC' | 'JWT' | 'BASIC';
 type WebhookResponseMode = 'SYNC' | 'ASYNC';
 type PaginationTypeValue = 'NONE' | 'OFFSET' | 'CURSOR' | 'PAGE' | 'LINK_HEADER';
 
@@ -351,7 +350,7 @@ interface FileUploadSourceConfig {
 
 interface WebhookSourceConfig {
     type: 'WEBHOOK';
-    authentication?: WebhookAuthMode;
+    authentication?: WebhookAuthType;
     secretCode?: string;
     idempotencyKeyField?: string;
     responseMode?: WebhookResponseMode;