@loxtep/sdk 0.7.21 → 0.7.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/README.md +17 -13
  3. package/dist/cli/commands/config-cmd.js +1 -1
  4. package/dist/cli/commands/connectors-cmd.d.ts +23 -0
  5. package/dist/cli/commands/connectors-cmd.d.ts.map +1 -0
  6. package/dist/cli/commands/connectors-cmd.js +32 -0
  7. package/dist/cli/commands/connectors-cmd.js.map +1 -0
  8. package/dist/cli/commands/deploy-cmd.d.ts +5 -1
  9. package/dist/cli/commands/deploy-cmd.d.ts.map +1 -1
  10. package/dist/cli/commands/deploy-cmd.js +25 -2
  11. package/dist/cli/commands/deploy-cmd.js.map +1 -1
  12. package/dist/cli/commands/ingest-cmd.d.ts +7 -2
  13. package/dist/cli/commands/ingest-cmd.d.ts.map +1 -1
  14. package/dist/cli/commands/ingest-cmd.js +120 -31
  15. package/dist/cli/commands/ingest-cmd.js.map +1 -1
  16. package/dist/cli/commands/lint-cmd.d.ts +17 -0
  17. package/dist/cli/commands/lint-cmd.d.ts.map +1 -0
  18. package/dist/cli/commands/lint-cmd.js +61 -0
  19. package/dist/cli/commands/lint-cmd.js.map +1 -0
  20. package/dist/cli/help.d.ts.map +1 -1
  21. package/dist/cli/help.js +11 -6
  22. package/dist/cli/help.js.map +1 -1
  23. package/dist/cli/index.js +24 -3
  24. package/dist/cli/index.js.map +1 -1
  25. package/dist/client/connectors.d.ts.map +1 -1
  26. package/dist/client/connectors.js +7 -1
  27. package/dist/client/connectors.js.map +1 -1
  28. package/dist/errors/parse-http.d.ts.map +1 -1
  29. package/dist/errors/parse-http.js +33 -10
  30. package/dist/errors/parse-http.js.map +1 -1
  31. package/dist/index.d.ts +5 -2
  32. package/dist/index.d.ts.map +1 -1
  33. package/dist/index.js +3 -1
  34. package/dist/index.js.map +1 -1
  35. package/dist/lib/entity-json-schemas/ajv-loader.cjs +17 -0
  36. package/dist/lib/entity-json-schemas/index.d.ts +5 -0
  37. package/dist/lib/entity-json-schemas/index.d.ts.map +1 -0
  38. package/dist/lib/entity-json-schemas/index.js +3 -0
  39. package/dist/lib/entity-json-schemas/index.js.map +1 -0
  40. package/dist/lib/entity-json-schemas/types.d.ts +21 -0
  41. package/dist/lib/entity-json-schemas/types.d.ts.map +1 -0
  42. package/dist/lib/entity-json-schemas/types.js +32 -0
  43. package/dist/lib/entity-json-schemas/types.js.map +1 -0
  44. package/dist/lib/entity-json-schemas/validate-entity.d.ts +39 -0
  45. package/dist/lib/entity-json-schemas/validate-entity.d.ts.map +1 -0
  46. package/dist/lib/entity-json-schemas/validate-entity.js +143 -0
  47. package/dist/lib/entity-json-schemas/validate-entity.js.map +1 -0
  48. package/dist/lib/sdk-ingest-bundle.d.ts +60 -4
  49. package/dist/lib/sdk-ingest-bundle.d.ts.map +1 -1
  50. package/dist/lib/sdk-ingest-bundle.js +144 -51
  51. package/dist/lib/sdk-ingest-bundle.js.map +1 -1
  52. package/dist/lib/workspace-lint.d.ts +28 -0
  53. package/dist/lib/workspace-lint.d.ts.map +1 -0
  54. package/dist/lib/workspace-lint.js +199 -0
  55. package/dist/lib/workspace-lint.js.map +1 -0
  56. package/docs/getting-started.md +6 -16
  57. package/docs/quick-reference.md +9 -9
  58. package/docs/sdk-first-ingest.md +64 -227
  59. package/package.json +6 -2
  60. package/schemas/entity-json-schemas/SCHEMA_SYNC.md +18 -0
  61. package/schemas/entity-json-schemas/connection.json +310 -0
  62. package/schemas/entity-json-schemas/connector.json +188 -0
  63. package/schemas/entity-json-schemas/container.json +71 -0
  64. package/schemas/entity-json-schemas/contract.json +228 -0
  65. package/schemas/entity-json-schemas/data-product.json +163 -0
  66. package/schemas/entity-json-schemas/domain.json +74 -0
  67. package/schemas/entity-json-schemas/export.json +126 -0
  68. package/schemas/entity-json-schemas/odps-product.json +150 -0
  69. package/schemas/entity-json-schemas/quality-rule.json +115 -0
  70. package/schemas/entity-json-schemas/schema.json +206 -0
  71. package/schemas/entity-json-schemas/transformation.json +166 -0
  72. package/schemas/entity-json-schemas/validation.json +152 -0
  73. package/schemas/entity-json-schemas/workflow-graph.json +116 -0
  74. package/schemas/entity-json-schemas/workflow.json +193 -0
  75. package/docs/examples/generate-ingest-bundle.mjs +0 -133
  76. package/docs/examples/write-events.mjs +0 -54
@@ -0,0 +1,228 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "$id": "https://loxtep.io/schemas/entity/contract.json",
4
+ "title": "Contract Entity",
5
+ "description": "JSON Schema for Contract entity in customer workspace. Contracts are versioned agreements that define guarantees (SLA/SLO) for a Data Product.",
6
+ "type": "object",
7
+ "required": [
8
+ "contract_id",
9
+ "organization_id",
10
+ "data_product_id",
11
+ "name",
12
+ "version",
13
+ "status",
14
+ "created_at",
15
+ "updated_at"
16
+ ],
17
+ "properties": {
18
+ "contract_id": {
19
+ "type": "string",
20
+ "format": "uuid",
21
+ "description": "Unique identifier for the contract"
22
+ },
23
+ "organization_id": {
24
+ "type": "string",
25
+ "format": "uuid",
26
+ "description": "Organization that owns this contract"
27
+ },
28
+ "data_product_id": {
29
+ "type": "string",
30
+ "format": "uuid",
31
+ "description": "Reference to the data product this contract applies to"
32
+ },
33
+ "name": {
34
+ "type": "string",
35
+ "minLength": 1,
36
+ "maxLength": 255,
37
+ "description": "Contract name"
38
+ },
39
+ "description": {
40
+ "type": ["string", "null"],
41
+ "maxLength": 2000,
42
+ "description": "Contract description"
43
+ },
44
+ "version": {
45
+ "type": "string",
46
+ "pattern": "^\\d+\\.\\d+\\.\\d+$",
47
+ "description": "Contract version (semantic versioning, e.g., 1.0.0)"
48
+ },
49
+ "status": {
50
+ "type": "string",
51
+ "enum": ["draft", "proposed", "active", "deprecated", "violated"],
52
+ "description": "Contract status"
53
+ },
54
+ "schema_ref": {
55
+ "type": ["object", "null"],
56
+ "description": "Reference to the schema version this contract applies to",
57
+ "properties": {
58
+ "schema_version_id": {
59
+ "type": "string",
60
+ "format": "uuid"
61
+ },
62
+ "version": {
63
+ "type": "string"
64
+ },
65
+ "format": {
66
+ "type": "string",
67
+ "enum": ["json-schema", "avro", "protobuf", "custom"]
68
+ }
69
+ },
70
+ "additionalProperties": false
71
+ },
72
+ "guarantees": {
73
+ "type": "object",
74
+ "properties": {
75
+ "availability": {
76
+ "type": ["string", "null"],
77
+ "description": "Availability guarantee (e.g., '99.9%')"
78
+ },
79
+ "freshness": {
80
+ "type": ["string", "null"],
81
+ "description": "Freshness guarantee (e.g., '< 5 minutes')"
82
+ },
83
+ "completeness": {
84
+ "type": ["string", "null"],
85
+ "description": "Completeness guarantee (e.g., '99%')"
86
+ },
87
+ "accuracy": {
88
+ "type": ["string", "null"],
89
+ "description": "Accuracy guarantee (e.g., '99.5%')"
90
+ }
91
+ },
92
+ "additionalProperties": false,
93
+ "default": {}
94
+ },
95
+ "sla_definitions": {
96
+ "type": "array",
97
+ "items": {
98
+ "type": "object",
99
+ "required": ["metric_type", "target_value"],
100
+ "properties": {
101
+ "metric_type": {
102
+ "type": "string",
103
+ "enum": [
104
+ "availability",
105
+ "freshness",
106
+ "completeness",
107
+ "accuracy",
108
+ "response_time",
109
+ "throughput"
110
+ ]
111
+ },
112
+ "target_value": {
113
+ "type": "number"
114
+ },
115
+ "unit": {
116
+ "type": ["string", "null"]
117
+ },
118
+ "measurement_window": {
119
+ "type": ["string", "null"]
120
+ },
121
+ "description": {
122
+ "type": ["string", "null"]
123
+ }
124
+ },
125
+ "additionalProperties": false
126
+ },
127
+ "default": [],
128
+ "description": "Detailed SLA definitions"
129
+ },
130
+ "sla": {
131
+ "type": "object",
132
+ "properties": {
133
+ "response_time": {
134
+ "type": ["string", "null"],
135
+ "description": "Response time SLA (e.g., '< 100ms p95')"
136
+ },
137
+ "throughput": {
138
+ "type": ["string", "null"],
139
+ "description": "Throughput SLA (e.g., '10000 events/sec')"
140
+ },
141
+ "support_level": {
142
+ "type": ["string", "null"],
143
+ "description": "Support level (e.g., '24x7', 'business_hours')"
144
+ },
145
+ "escalation_contact": {
146
+ "type": ["string", "null"],
147
+ "format": "email"
148
+ }
149
+ },
150
+ "additionalProperties": false,
151
+ "default": {}
152
+ },
153
+ "terms": {
154
+ "type": ["object", "null"],
155
+ "properties": {
156
+ "data_retention_days": {
157
+ "type": ["integer", "null"],
158
+ "minimum": 1
159
+ },
160
+ "usage_restrictions": {
161
+ "type": ["array", "null"],
162
+ "items": {
163
+ "type": "string"
164
+ }
165
+ },
166
+ "compliance_requirements": {
167
+ "type": ["array", "null"],
168
+ "items": {
169
+ "type": "string"
170
+ }
171
+ }
172
+ },
173
+ "additionalProperties": false
174
+ },
175
+ "effective_date": {
176
+ "type": ["string", "null"],
177
+ "format": "date-time",
178
+ "description": "ISO 8601 timestamp when contract becomes effective"
179
+ },
180
+ "expiration_date": {
181
+ "type": ["string", "null"],
182
+ "format": "date-time",
183
+ "description": "ISO 8601 timestamp when contract expires"
184
+ },
185
+ "metadata": {
186
+ "type": "object",
187
+ "additionalProperties": true,
188
+ "default": {},
189
+ "description": "Additional metadata for the contract"
190
+ },
191
+ "created_by": {
192
+ "type": ["string", "null"],
193
+ "format": "uuid",
194
+ "description": "User ID who created the contract"
195
+ },
196
+ "updated_by": {
197
+ "type": ["string", "null"],
198
+ "format": "uuid",
199
+ "description": "User ID who last updated the contract"
200
+ },
201
+ "approved_by": {
202
+ "type": ["string", "null"],
203
+ "format": "uuid",
204
+ "description": "User ID who approved the contract"
205
+ },
206
+ "approved_at": {
207
+ "type": ["string", "null"],
208
+ "format": "date-time",
209
+ "description": "ISO 8601 timestamp when contract was approved"
210
+ },
211
+ "created_at": {
212
+ "type": "string",
213
+ "format": "date-time",
214
+ "description": "ISO 8601 timestamp when contract was created"
215
+ },
216
+ "updated_at": {
217
+ "type": "string",
218
+ "format": "date-time",
219
+ "description": "ISO 8601 timestamp when contract was last updated"
220
+ },
221
+ "deleted_at": {
222
+ "type": ["string", "null"],
223
+ "format": "date-time",
224
+ "description": "ISO 8601 timestamp when contract was deleted (soft delete)"
225
+ }
226
+ },
227
+ "additionalProperties": false
228
+ }
@@ -0,0 +1,163 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "$id": "https://loxtep.io/schemas/entity/data-product.json",
4
+ "title": "Data Product Entity",
5
+ "description": "JSON Schema for Data Product entity in customer workspace (Studio / S3 graph). Flow-graph fields (workflow_id, upstream_entity_id, etc.) describe placement in the workflow canvas. The canonical interoperable catalog and API representation is ODPS v4.1 (Open Data Products Standard): include `odps_document` aligned with https://opendataproducts.org/v4.1/schema/odps.json — see `odps-product.json` and .kiro/specs/odps-data-product-standard in the Loxtep repo.",
6
+ "type": "object",
7
+ "required": [
8
+ "data_product_id",
9
+ "organization_id",
10
+ "workflow_id",
11
+ "name",
12
+ "status",
13
+ "created_at",
14
+ "updated_at"
15
+ ],
16
+ "properties": {
17
+ "data_product_id": {
18
+ "type": "string",
19
+ "format": "uuid",
20
+ "description": "Unique identifier for the data product"
21
+ },
22
+ "organization_id": {
23
+ "type": "string",
24
+ "format": "uuid",
25
+ "description": "Organization that owns this data product"
26
+ },
27
+ "workflow_id": {
28
+ "type": "string",
29
+ "format": "uuid",
30
+ "description": "Reference to the workflow that produces this data product"
31
+ },
32
+ "domain_id": {
33
+ "type": ["string", "null"],
34
+ "format": "uuid",
35
+ "description": "Optional reference to the domain this data product belongs to"
36
+ },
37
+ "upstream_entity_id": {
38
+ "type": ["string", "null"],
39
+ "format": "uuid",
40
+ "description": "Reference to the upstream entity in the flow graph (last transformation, validation, or connection before this data product; source data product in an enrichment flow)"
41
+ },
42
+ "upstream_entity_type": {
43
+ "type": ["string", "null"],
44
+ "enum": [
45
+ "connections",
46
+ "transformations",
47
+ "validations",
48
+ "data-products",
49
+ null
50
+ ],
51
+ "description": "Entity type of the upstream entity"
52
+ },
53
+ "name": {
54
+ "type": "string",
55
+ "minLength": 1,
56
+ "maxLength": 255,
57
+ "description": "Data product name"
58
+ },
59
+ "description": {
60
+ "type": ["string", "null"],
61
+ "maxLength": 2000,
62
+ "description": "Data product description"
63
+ },
64
+ "owner": {
65
+ "type": "object",
66
+ "required": ["user_id"],
67
+ "properties": {
68
+ "user_id": {
69
+ "type": "string",
70
+ "format": "uuid",
71
+ "description": "User ID of the data product owner"
72
+ },
73
+ "team": {
74
+ "type": ["string", "null"],
75
+ "description": "Team name (optional)"
76
+ }
77
+ },
78
+ "additionalProperties": false
79
+ },
80
+ "status": {
81
+ "type": "string",
82
+ "enum": ["draft", "active", "deprecated", "archived"],
83
+ "description": "Data product status"
84
+ },
85
+ "governance": {
86
+ "type": "object",
87
+ "required": ["classification"],
88
+ "properties": {
89
+ "classification": {
90
+ "type": "string",
91
+ "enum": ["public", "internal", "confidential", "restricted"]
92
+ },
93
+ "pii_fields": {
94
+ "type": "array",
95
+ "items": {
96
+ "type": "string"
97
+ },
98
+ "default": []
99
+ },
100
+ "compliance_requirements": {
101
+ "type": "array",
102
+ "items": {
103
+ "type": "string",
104
+ "enum": ["GDPR", "CCPA", "HIPAA", "SOX", "PCI_DSS"]
105
+ },
106
+ "default": []
107
+ },
108
+ "tags": {
109
+ "type": "array",
110
+ "items": {
111
+ "type": "string"
112
+ },
113
+ "default": []
114
+ }
115
+ },
116
+ "additionalProperties": false,
117
+ "default": {
118
+ "classification": "internal",
119
+ "pii_fields": [],
120
+ "compliance_requirements": [],
121
+ "tags": []
122
+ }
123
+ },
124
+ "metadata": {
125
+ "type": "object",
126
+ "additionalProperties": true,
127
+ "default": {},
128
+ "description": "Additional metadata for the data product"
129
+ },
130
+ "created_by": {
131
+ "type": ["string", "null"],
132
+ "format": "uuid",
133
+ "description": "User ID who created the data product"
134
+ },
135
+ "updated_by": {
136
+ "type": ["string", "null"],
137
+ "format": "uuid",
138
+ "description": "User ID who last updated the data product"
139
+ },
140
+ "created_at": {
141
+ "type": "string",
142
+ "format": "date-time",
143
+ "description": "ISO 8601 timestamp when data product was created"
144
+ },
145
+ "updated_at": {
146
+ "type": "string",
147
+ "format": "date-time",
148
+ "description": "ISO 8601 timestamp when data product was last updated"
149
+ },
150
+ "deleted_at": {
151
+ "type": ["string", "null"],
152
+ "format": "date-time",
153
+ "description": "ISO 8601 timestamp when data product was deleted (soft delete)"
154
+ },
155
+ "odps_document": {
156
+ "description": "Canonical ODPS v4.1 document (`{ product: ... }`). Preferred for platform catalog, governance, and ecosystem interoperability. Validated in the dataproducts API; Loxtep-specific fields under product.custom.loxtep.",
157
+ "allOf": [
158
+ { "$ref": "https://loxtep.io/schemas/entity/odps-product.json" }
159
+ ]
160
+ }
161
+ },
162
+ "additionalProperties": false
163
+ }
@@ -0,0 +1,74 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "$id": "https://loxtep.io/schemas/entity/domain.json",
4
+ "title": "Domain Entity",
5
+ "description": "JSON Schema for Domain entity in customer workspace. Domains are logical, business-oriented subdivisions for organizing Data Products.",
6
+ "type": "object",
7
+ "required": [
8
+ "domain_id",
9
+ "organization_id",
10
+ "name",
11
+ "created_at",
12
+ "updated_at"
13
+ ],
14
+ "properties": {
15
+ "domain_id": {
16
+ "type": "string",
17
+ "format": "uuid",
18
+ "description": "Unique identifier for the domain"
19
+ },
20
+ "organization_id": {
21
+ "type": "string",
22
+ "format": "uuid",
23
+ "description": "Organization that owns this domain"
24
+ },
25
+ "name": {
26
+ "type": "string",
27
+ "minLength": 1,
28
+ "maxLength": 255,
29
+ "description": "Domain name"
30
+ },
31
+ "description": {
32
+ "type": ["string", "null"],
33
+ "maxLength": 1000,
34
+ "description": "Domain description"
35
+ },
36
+ "owner_user_id": {
37
+ "type": ["string", "null"],
38
+ "format": "uuid",
39
+ "description": "User ID of the domain owner"
40
+ },
41
+ "parent_domain_id": {
42
+ "type": ["string", "null"],
43
+ "format": "uuid",
44
+ "description": "Parent domain ID for hierarchical organization"
45
+ },
46
+ "is_council": {
47
+ "type": "boolean",
48
+ "default": false,
49
+ "description": "True if this is the Data Council domain"
50
+ },
51
+ "metadata": {
52
+ "type": "object",
53
+ "additionalProperties": true,
54
+ "default": {},
55
+ "description": "Additional metadata for the domain"
56
+ },
57
+ "created_at": {
58
+ "type": "string",
59
+ "format": "date-time",
60
+ "description": "ISO 8601 timestamp when domain was created"
61
+ },
62
+ "updated_at": {
63
+ "type": "string",
64
+ "format": "date-time",
65
+ "description": "ISO 8601 timestamp when domain was last updated"
66
+ },
67
+ "deleted_at": {
68
+ "type": ["string", "null"],
69
+ "format": "date-time",
70
+ "description": "ISO 8601 timestamp when domain was deleted (soft delete)"
71
+ }
72
+ },
73
+ "additionalProperties": false
74
+ }
@@ -0,0 +1,126 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "$id": "https://loxtep.io/schemas/entity/export.json",
4
+ "title": "Export Entity",
5
+ "description": "JSON Schema for Export entity in customer workspace. Exports define reusable export configurations for data products.",
6
+ "type": "object",
7
+ "required": [
8
+ "export_id",
9
+ "organization_id",
10
+ "data_product_id",
11
+ "name",
12
+ "format",
13
+ "compression",
14
+ "destination",
15
+ "created_at",
16
+ "updated_at"
17
+ ],
18
+ "properties": {
19
+ "export_id": {
20
+ "type": "string",
21
+ "format": "uuid",
22
+ "description": "Unique identifier for the export configuration"
23
+ },
24
+ "organization_id": {
25
+ "type": "string",
26
+ "format": "uuid",
27
+ "description": "Organization that owns this export"
28
+ },
29
+ "data_product_id": {
30
+ "type": "string",
31
+ "format": "uuid",
32
+ "description": "Reference to the data product this export applies to"
33
+ },
34
+ "name": {
35
+ "type": "string",
36
+ "minLength": 1,
37
+ "maxLength": 255,
38
+ "description": "Export configuration name"
39
+ },
40
+ "description": {
41
+ "type": ["string", "null"],
42
+ "maxLength": 1000,
43
+ "description": "Export description"
44
+ },
45
+ "format": {
46
+ "type": "string",
47
+ "enum": ["csv", "parquet", "json", "jsonl", "avro"],
48
+ "description": "Export file format"
49
+ },
50
+ "compression": {
51
+ "type": "string",
52
+ "enum": ["none", "gzip", "snappy", "zstd"],
53
+ "default": "none",
54
+ "description": "Compression type"
55
+ },
56
+ "destination": {
57
+ "type": "string",
58
+ "enum": ["download", "s3", "gcs", "azure_blob"],
59
+ "description": "Export destination type"
60
+ },
61
+ "destination_config": {
62
+ "type": "object",
63
+ "additionalProperties": true,
64
+ "default": {},
65
+ "description": "Destination-specific configuration (S3 bucket, GCS path, etc.)"
66
+ },
67
+ "fields": {
68
+ "type": "array",
69
+ "items": {
70
+ "type": "string",
71
+ "minLength": 1
72
+ },
73
+ "minItems": 0,
74
+ "description": "Field names to include in export (empty = all fields)"
75
+ },
76
+ "date_filter": {
77
+ "type": "object",
78
+ "additionalProperties": true,
79
+ "default": {},
80
+ "description": "Date range filter configuration"
81
+ },
82
+ "schedule": {
83
+ "type": ["object", "null"],
84
+ "additionalProperties": true,
85
+ "description": "Schedule configuration for recurring exports (cron expression)"
86
+ },
87
+ "options": {
88
+ "type": "object",
89
+ "additionalProperties": true,
90
+ "default": {},
91
+ "description": "Additional export options (delimiter, headers, etc.)"
92
+ },
93
+ "metadata": {
94
+ "type": "object",
95
+ "additionalProperties": true,
96
+ "default": {},
97
+ "description": "Additional metadata for the export"
98
+ },
99
+ "created_by": {
100
+ "type": ["string", "null"],
101
+ "format": "uuid",
102
+ "description": "User ID who created the export"
103
+ },
104
+ "updated_by": {
105
+ "type": ["string", "null"],
106
+ "format": "uuid",
107
+ "description": "User ID who last updated the export"
108
+ },
109
+ "created_at": {
110
+ "type": "string",
111
+ "format": "date-time",
112
+ "description": "ISO 8601 timestamp when export was created"
113
+ },
114
+ "updated_at": {
115
+ "type": "string",
116
+ "format": "date-time",
117
+ "description": "ISO 8601 timestamp when export was last updated"
118
+ },
119
+ "deleted_at": {
120
+ "type": ["string", "null"],
121
+ "format": "date-time",
122
+ "description": "ISO 8601 timestamp when export was deleted (soft delete)"
123
+ }
124
+ },
125
+ "additionalProperties": false
126
+ }