@loxtep/sdk 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/cli/commands/delivery-cmd.d.ts +1 -0
- package/dist/cli/commands/delivery-cmd.d.ts.map +1 -1
- package/dist/cli/commands/delivery-cmd.js +60 -44
- package/dist/cli/commands/delivery-cmd.js.map +1 -1
- package/dist/cli/commands/ingest-cmd.d.ts +1 -0
- package/dist/cli/commands/ingest-cmd.d.ts.map +1 -1
- package/dist/cli/commands/ingest-cmd.js +37 -24
- package/dist/cli/commands/ingest-cmd.js.map +1 -1
- package/dist/cli/commands/transform-cmd.d.ts +1 -0
- package/dist/cli/commands/transform-cmd.d.ts.map +1 -1
- package/dist/cli/commands/transform-cmd.js +36 -23
- package/dist/cli/commands/transform-cmd.js.map +1 -1
- package/dist/cli/help.d.ts.map +1 -1
- package/dist/cli/help.js +5 -3
- package/dist/cli/help.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +565 -522
- package/dist/cli/index.js.map +1 -1
- package/dist/client/data-products.d.ts +10 -8
- package/dist/client/data-products.d.ts.map +1 -1
- package/dist/client/data-products.js +142 -93
- package/dist/client/data-products.js.map +1 -1
- package/dist/client/flow-types.d.ts +16 -6
- package/dist/client/flow-types.d.ts.map +1 -1
- package/dist/client/flow-types.js +6 -1
- package/dist/client/flow-types.js.map +1 -1
- package/dist/client/index.d.ts +89 -88
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +13 -12
- package/dist/client/index.js.map +1 -1
- package/dist/client/queues.d.ts +5 -3
- package/dist/client/queues.d.ts.map +1 -1
- package/dist/client/queues.js +26 -21
- package/dist/client/queues.js.map +1 -1
- package/dist/client/workflows.d.ts +7 -7
- package/dist/client/workflows.d.ts.map +1 -1
- package/dist/client/workflows.js +71 -63
- package/dist/client/workflows.js.map +1 -1
- package/dist/lib/sdk-ingest-bundle.d.ts +5 -1
- package/dist/lib/sdk-ingest-bundle.d.ts.map +1 -1
- package/dist/lib/sdk-ingest-bundle.js +38 -29
- package/dist/lib/sdk-ingest-bundle.js.map +1 -1
- package/dist/rstreams/event-bridge.d.ts +4 -4
- package/dist/rstreams/event-bridge.d.ts.map +1 -1
- package/dist/rstreams/event-bridge.js +23 -16
- package/dist/rstreams/event-bridge.js.map +1 -1
- package/docs/event-writer-transport-modes.md +47 -0
- package/docs/sdk-first-ingest.md +14 -12
- package/package.json +1 -1
- package/schemas/entity-json-schemas/SCHEMA_SYNC.md +1 -1
- package/schemas/entity-json-schemas/SOURCE.json +5 -4
- package/schemas/entity-json-schemas/connection.json +35 -0
- package/schemas/entity-json-schemas/data-product.json +16 -0
- package/schemas/entity-json-schemas/grant.json +146 -0
- package/schemas/entity-json-schemas/transformation.json +19 -0
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,13 @@ and this project adheres to
|
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- Event writers and the `ingest create`, `transform create`, and
|
|
14
|
+
`delivery create` commands accept `transport_mode`: `default` is the durable,
|
|
15
|
+
recommended mode; `fast` minimizes latency but may cost more; and `volume`
|
|
16
|
+
supports high throughput with delivery lag.
|
|
17
|
+
|
|
11
18
|
## [0.12.0] - 2026-10-30
|
|
12
19
|
|
|
13
20
|
### Added
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"delivery-cmd.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/delivery-cmd.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"delivery-cmd.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/delivery-cmd.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH,MAAM,WAAW,kBAAkB;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,2EAA2E;AAC3E,eAAO,MAAM,oBAAoB,yCAAyC,CAAC;AAC3E,eAAO,MAAM,sBAAsB,uBAAuB,CAAC;AAgB3D;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE;IACN,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,EACD,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,IAAI,CAAC,CAuPf"}
|
|
@@ -2,16 +2,17 @@
|
|
|
2
2
|
* CLI: loxtep delivery create — delivery workflow from the connector-delivery
|
|
3
3
|
* project template: data product → transform → outbound connection.
|
|
4
4
|
*/
|
|
5
|
-
import { existsSync, mkdirSync, writeFileSync } from
|
|
6
|
-
import { dirname, join } from
|
|
7
|
-
import { randomUUID } from
|
|
8
|
-
import { findProjectDir } from
|
|
9
|
-
import { requireCliClient } from
|
|
10
|
-
import { formatLintResult, runLintCheck } from
|
|
11
|
-
import { toLocalConnectorEntity } from
|
|
5
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
6
|
+
import { dirname, join } from "node:path";
|
|
7
|
+
import { randomUUID } from "node:crypto";
|
|
8
|
+
import { findProjectDir } from "../project-context.js";
|
|
9
|
+
import { requireCliClient } from "../create-cli-client.js";
|
|
10
|
+
import { formatLintResult, runLintCheck } from "./lint-cmd.js";
|
|
11
|
+
import { toLocalConnectorEntity } from "../../lib/sdk-ingest-bundle.js";
|
|
12
|
+
import { parseEventWriterTransportMode, } from "../../client/flow-types.js";
|
|
12
13
|
/** Stable template id for connector-delivery (loxtep-project-template). */
|
|
13
|
-
export const DELIVERY_TEMPLATE_ID =
|
|
14
|
-
export const DELIVERY_TEMPLATE_SLUG =
|
|
14
|
+
export const DELIVERY_TEMPLATE_ID = "00000000-0000-4000-8000-0000000000a3";
|
|
15
|
+
export const DELIVERY_TEMPLATE_SLUG = "connector-delivery";
|
|
15
16
|
function writePackageFiles(projectDir, files) {
|
|
16
17
|
const written = [];
|
|
17
18
|
for (const [relPath, entity] of Object.entries(files)) {
|
|
@@ -34,42 +35,52 @@ export async function runDeliveryCreate(params, options = {}) {
|
|
|
34
35
|
const cwd = options.cwd ?? process.cwd();
|
|
35
36
|
const projectDir = findProjectDir(cwd) ?? cwd;
|
|
36
37
|
const { client, config } = await requireCliClient(options);
|
|
38
|
+
let transportMode;
|
|
39
|
+
try {
|
|
40
|
+
transportMode = parseEventWriterTransportMode(params.transport_mode);
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
44
|
+
process.exitCode = 1;
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
37
47
|
const projectId = params.project_id ?? config.project_id;
|
|
38
48
|
if (!projectId) {
|
|
39
|
-
console.error(
|
|
49
|
+
console.error("Missing project_id. Run `loxtep init` and attach a project first.");
|
|
40
50
|
process.exitCode = 1;
|
|
41
51
|
return;
|
|
42
52
|
}
|
|
43
53
|
if (!params.from?.trim()) {
|
|
44
|
-
console.error(
|
|
54
|
+
console.error("Usage: loxtep delivery create --from <dp-name> --connector-id <id> [--table schema.table] [--name …]");
|
|
45
55
|
process.exitCode = 1;
|
|
46
56
|
return;
|
|
47
57
|
}
|
|
48
58
|
if (!params.connector_id?.trim()) {
|
|
49
|
-
console.error(
|
|
59
|
+
console.error("Missing --connector-id (target connector at the delivery tail).");
|
|
50
60
|
process.exitCode = 1;
|
|
51
61
|
return;
|
|
52
62
|
}
|
|
53
63
|
const user = await client.session.get_current_user();
|
|
54
64
|
const organizationId = user.organization_id ?? client.organization_id;
|
|
55
65
|
if (!organizationId) {
|
|
56
|
-
console.error(
|
|
66
|
+
console.error("Missing organization_id in session.");
|
|
57
67
|
process.exitCode = 1;
|
|
58
68
|
return;
|
|
59
69
|
}
|
|
60
70
|
// Verify connector exists and mirror it under connectors/ for workspace lint.
|
|
61
71
|
const connector = await client.connect.connectors.get(params.connector_id);
|
|
62
|
-
const localConnectorPath = join(projectDir,
|
|
72
|
+
const localConnectorPath = join(projectDir, "connectors", `${connector.connector_id}.json`);
|
|
63
73
|
const includeConnectorFile = !existsSync(localConnectorPath);
|
|
64
|
-
const connectionType = (typeof connector.connector_type ===
|
|
65
|
-
|
|
74
|
+
const connectionType = (typeof connector.connector_type === "string" &&
|
|
75
|
+
connector.connector_type.trim()) ||
|
|
76
|
+
"webhook";
|
|
66
77
|
let domainId = params.domain_id;
|
|
67
78
|
if (!domainId) {
|
|
68
79
|
const domains = await client.define.domains.list({ page_size: 1 });
|
|
69
80
|
domainId = domains.items?.[0]?.domain_id;
|
|
70
81
|
}
|
|
71
82
|
if (!domainId) {
|
|
72
|
-
console.error(
|
|
83
|
+
console.error("Missing domain_id. Pass --domain-id or create a domain first.");
|
|
73
84
|
process.exitCode = 1;
|
|
74
85
|
return;
|
|
75
86
|
}
|
|
@@ -79,9 +90,9 @@ export async function runDeliveryCreate(params, options = {}) {
|
|
|
79
90
|
const transformId = randomUUID();
|
|
80
91
|
const targetConnectionId = randomUUID();
|
|
81
92
|
const now = new Date().toISOString();
|
|
82
|
-
const table = params.table?.trim() ||
|
|
83
|
-
const schema = params.schema?.trim() ||
|
|
84
|
-
const primaryKey = params.primary_key?.trim() ||
|
|
93
|
+
const table = params.table?.trim() || "";
|
|
94
|
+
const schema = params.schema?.trim() || "public";
|
|
95
|
+
const primaryKey = params.primary_key?.trim() || "_id";
|
|
85
96
|
const files = {};
|
|
86
97
|
if (includeConnectorFile) {
|
|
87
98
|
files[`connectors/${connector.connector_id}.json`] = toLocalConnectorEntity(connector, now);
|
|
@@ -93,14 +104,14 @@ export async function runDeliveryCreate(params, options = {}) {
|
|
|
93
104
|
name: workflowName,
|
|
94
105
|
description: `Delivery workflow: ${params.from} → ${connectionType} target`,
|
|
95
106
|
template_id: DELIVERY_TEMPLATE_ID,
|
|
96
|
-
workflow_type:
|
|
107
|
+
workflow_type: "delivery",
|
|
97
108
|
domain_id: domainId,
|
|
98
|
-
status:
|
|
109
|
+
status: "active",
|
|
99
110
|
configuration: {
|
|
100
|
-
activation:
|
|
111
|
+
activation: "event",
|
|
101
112
|
},
|
|
102
113
|
metadata: {
|
|
103
|
-
stage:
|
|
114
|
+
stage: "delivery",
|
|
104
115
|
template_slug: DELIVERY_TEMPLATE_SLUG,
|
|
105
116
|
upstream_data_product: params.from,
|
|
106
117
|
transform_id: transformId,
|
|
@@ -113,17 +124,18 @@ export async function runDeliveryCreate(params, options = {}) {
|
|
|
113
124
|
organization_id: organizationId,
|
|
114
125
|
project_id: projectId,
|
|
115
126
|
workflow_id: workflowId,
|
|
116
|
-
key:
|
|
127
|
+
key: "data-product-source",
|
|
117
128
|
name: `From ${params.from}`,
|
|
118
|
-
type:
|
|
119
|
-
connector_type:
|
|
120
|
-
status:
|
|
129
|
+
type: "data-product-trigger",
|
|
130
|
+
connector_type: "data-product-trigger",
|
|
131
|
+
status: "active",
|
|
121
132
|
configuration: {
|
|
122
133
|
source_data_product_name: params.from,
|
|
123
134
|
},
|
|
135
|
+
runtime_settings: { event_writer: { transport_mode: transportMode } },
|
|
124
136
|
metadata: {
|
|
125
137
|
template_slug: DELIVERY_TEMPLATE_SLUG,
|
|
126
|
-
role:
|
|
138
|
+
role: "source",
|
|
127
139
|
},
|
|
128
140
|
created_at: now,
|
|
129
141
|
updated_at: now,
|
|
@@ -133,26 +145,27 @@ export async function runDeliveryCreate(params, options = {}) {
|
|
|
133
145
|
organization_id: organizationId,
|
|
134
146
|
project_id: projectId,
|
|
135
147
|
workflow_id: workflowId,
|
|
136
|
-
name:
|
|
137
|
-
transform_type:
|
|
148
|
+
name: "Map to destination columns",
|
|
149
|
+
transform_type: "select_fields",
|
|
138
150
|
operation_config: {
|
|
139
151
|
fields: [],
|
|
140
152
|
},
|
|
153
|
+
runtime_settings: { event_writer: { transport_mode: transportMode } },
|
|
141
154
|
upstream_entity_id: triggerConnectionId,
|
|
142
|
-
upstream_entity_type:
|
|
155
|
+
upstream_entity_type: "connections",
|
|
143
156
|
metadata: {
|
|
144
157
|
template_slug: DELIVERY_TEMPLATE_SLUG,
|
|
145
|
-
notes:
|
|
158
|
+
notes: "Author field→column mapping here. Outbound bot expects a flat destination-shaped row.",
|
|
146
159
|
},
|
|
147
160
|
created_at: now,
|
|
148
161
|
updated_at: now,
|
|
149
162
|
};
|
|
150
163
|
const deliveryConfig = {
|
|
151
|
-
write_mode:
|
|
164
|
+
write_mode: "upsert",
|
|
152
165
|
primary_key: [primaryKey],
|
|
153
166
|
batch_size: 100,
|
|
154
167
|
insert_batch_size: 1000,
|
|
155
|
-
conflict_update: { default:
|
|
168
|
+
conflict_update: { default: "excluded" },
|
|
156
169
|
};
|
|
157
170
|
if (table) {
|
|
158
171
|
deliveryConfig.table = table;
|
|
@@ -164,20 +177,21 @@ export async function runDeliveryCreate(params, options = {}) {
|
|
|
164
177
|
project_id: projectId,
|
|
165
178
|
workflow_id: workflowId,
|
|
166
179
|
connector_id: params.connector_id,
|
|
167
|
-
key:
|
|
168
|
-
name:
|
|
180
|
+
key: "delivery-target",
|
|
181
|
+
name: "Delivery Target",
|
|
169
182
|
type: connectionType,
|
|
170
183
|
connector_type: connectionType,
|
|
171
|
-
status:
|
|
172
|
-
connector_mode:
|
|
184
|
+
status: "active",
|
|
185
|
+
connector_mode: "outbound",
|
|
173
186
|
configuration: {},
|
|
174
187
|
delivery_config: deliveryConfig,
|
|
188
|
+
runtime_settings: { event_writer: { transport_mode: transportMode } },
|
|
175
189
|
upstream_entity_id: transformId,
|
|
176
|
-
upstream_entity_type:
|
|
190
|
+
upstream_entity_type: "transformations",
|
|
177
191
|
metadata: {
|
|
178
192
|
template_slug: DELIVERY_TEMPLATE_SLUG,
|
|
179
|
-
role:
|
|
180
|
-
notes:
|
|
193
|
+
role: "target",
|
|
194
|
+
notes: "Auth/transport come from the org connector. Write policy (table, batch sizes, conflict_update) lives on delivery_config. List tables via `loxtep connectors list-entities <connector_id>`.",
|
|
181
195
|
},
|
|
182
196
|
created_at: now,
|
|
183
197
|
updated_at: now,
|
|
@@ -186,11 +200,12 @@ export async function runDeliveryCreate(params, options = {}) {
|
|
|
186
200
|
console.log(JSON.stringify({
|
|
187
201
|
dry_run: true,
|
|
188
202
|
workflow_id: workflowId,
|
|
189
|
-
workflow_type:
|
|
203
|
+
workflow_type: "delivery",
|
|
190
204
|
template_slug: DELIVERY_TEMPLATE_SLUG,
|
|
191
205
|
connector_id: params.connector_id,
|
|
192
206
|
transform_id: transformId,
|
|
193
207
|
target_connection_id: targetConnectionId,
|
|
208
|
+
transport_mode: transportMode,
|
|
194
209
|
files: Object.keys(files),
|
|
195
210
|
}, null, 2));
|
|
196
211
|
return;
|
|
@@ -206,11 +221,12 @@ export async function runDeliveryCreate(params, options = {}) {
|
|
|
206
221
|
}
|
|
207
222
|
console.log(JSON.stringify({
|
|
208
223
|
workflow_id: workflowId,
|
|
209
|
-
workflow_type:
|
|
224
|
+
workflow_type: "delivery",
|
|
210
225
|
template_slug: DELIVERY_TEMPLATE_SLUG,
|
|
211
226
|
connector_id: params.connector_id,
|
|
212
227
|
transform_id: transformId,
|
|
213
228
|
target_connection_id: targetConnectionId,
|
|
229
|
+
transport_mode: transportMode,
|
|
214
230
|
files: written,
|
|
215
231
|
}, null, 2));
|
|
216
232
|
console.error(`
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"delivery-cmd.js","sourceRoot":"","sources":["../../../src/cli/commands/delivery-cmd.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;
|
|
1
|
+
{"version":3,"file":"delivery-cmd.js","sourceRoot":"","sources":["../../../src/cli/commands/delivery-cmd.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EACL,6BAA6B,GAE9B,MAAM,4BAA4B,CAAC;AAUpC,2EAA2E;AAC3E,MAAM,CAAC,MAAM,oBAAoB,GAAG,sCAAsC,CAAC;AAC3E,MAAM,CAAC,MAAM,sBAAsB,GAAG,oBAAoB,CAAC;AAE3D,SAAS,iBAAiB,CACxB,UAAkB,EAClB,KAA8C;IAE9C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACvC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAYC,EACD,UAA8B,EAAE;IAEhC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;IAC9C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,aAAuC,CAAC;IAC5C,IAAI,CAAC;QACH,aAAa,GAAG,6BAA6B,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;IACzD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,mEAAmE,CACpE,CAAC;QACF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CACX,sGAAsG,CACvG,CAAC;QACF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CACX,iEAAiE,CAClE,CAAC;QACF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;IACrD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,CAAC;IACtE,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACrD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,8EAA8E;IAC9E,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAC3E,MAAM,kBAAkB,GAAG,IAAI,CAC7B,UAAU,EACV,YAAY,EACZ,GAAG,SAAS,CAAC,YAAY,OAAO,CACjC,CAAC;IACF,MAAM,oBAAoB,GAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;IAC7D,MAAM,cAAc,GAClB,CAAC,OAAO,SAAS,CAAC,cAAc,KAAK,QAAQ;QAC3C,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAClC,SAAS,CAAC;IAEZ,IAAI,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC;IAChC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACnE,QAAQ,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;IAC3C,CAAC;IACD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,CAAC,KAAK,CACX,+DAA+D,CAChE,CAAC;QACF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,WAAW,CAAC;IAC9D,MAAM,UAAU,GAAG,UAAU,EAAE,CAAC;IAChC,MAAM,mBAAmB,GAAG,UAAU,EAAE,CAAC;IACzC,MAAM,WAAW,GAAG,UAAU,EAAE,CAAC;IACjC,MAAM,kBAAkB,GAAG,UAAU,EAAE,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC;IACjD,MAAM,UAAU,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC;IAEvD,MAAM,KAAK,GAA4C,EAAE,CAAC;IAC1D,IAAI,oBAAoB,EAAE,CAAC;QACzB,KAAK,CAAC,cAAc,SAAS,CAAC,YAAY,OAAO,CAAC,GAAG,sBAAsB,CACzE,SAAS,EACT,GAAG,CACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa,UAAU,gBAAgB,CAAC,GAAG;QAC/C,WAAW,EAAE,UAAU;QACvB,eAAe,EAAE,cAAc;QAC/B,UAAU,EAAE,SAAS;QACrB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,sBAAsB,MAAM,CAAC,IAAI,MAAM,cAAc,SAAS;QAC3E,WAAW,EAAE,oBAAoB;QACjC,aAAa,EAAE,UAAU;QACzB,SAAS,EAAE,QAAQ;QACnB,MAAM,EAAE,QAAQ;QAChB,aAAa,EAAE;YACb,UAAU,EAAE,OAAO;SACpB;QACD,QAAQ,EAAE;YACR,KAAK,EAAE,UAAU;YACjB,aAAa,EAAE,sBAAsB;YACrC,qBAAqB,EAAE,MAAM,CAAC,IAAI;YAClC,YAAY,EAAE,WAAW;SAC1B;QACD,UAAU,EAAE,GAAG;QACf,UAAU,EAAE,GAAG;KAChB,CAAC;IAEF,KAAK,CAAC,aAAa,UAAU,gBAAgB,mBAAmB,OAAO,CAAC,GAAG;QACzE,aAAa,EAAE,mBAAmB;QAClC,eAAe,EAAE,cAAc;QAC/B,UAAU,EAAE,SAAS;QACrB,WAAW,EAAE,UAAU;QACvB,GAAG,EAAE,qBAAqB;QAC1B,IAAI,EAAE,QAAQ,MAAM,CAAC,IAAI,EAAE;QAC3B,IAAI,EAAE,sBAAsB;QAC5B,cAAc,EAAE,sBAAsB;QACtC,MAAM,EAAE,QAAQ;QAChB,aAAa,EAAE;YACb,wBAAwB,EAAE,MAAM,CAAC,IAAI;SACtC;QACD,gBAAgB,EAAE,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,aAAa,EAAE,EAAE;QACrE,QAAQ,EAAE;YACR,aAAa,EAAE,sBAAsB;YACrC,IAAI,EAAE,QAAQ;SACf;QACD,UAAU,EAAE,GAAG;QACf,UAAU,EAAE,GAAG;KAChB,CAAC;IAEF,KAAK,CAAC,aAAa,UAAU,oBAAoB,WAAW,OAAO,CAAC,GAAG;QACrE,iBAAiB,EAAE,WAAW;QAC9B,eAAe,EAAE,cAAc;QAC/B,UAAU,EAAE,SAAS;QACrB,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,4BAA4B;QAClC,cAAc,EAAE,eAAe;QAC/B,gBAAgB,EAAE;YAChB,MAAM,EAAE,EAAE;SACX;QACD,gBAAgB,EAAE,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,aAAa,EAAE,EAAE;QACrE,kBAAkB,EAAE,mBAAmB;QACvC,oBAAoB,EAAE,aAAa;QACnC,QAAQ,EAAE;YACR,aAAa,EAAE,sBAAsB;YACrC,KAAK,EACH,uFAAuF;SAC1F;QACD,UAAU,EAAE,GAAG;QACf,UAAU,EAAE,GAAG;KAChB,CAAC;IAEF,MAAM,cAAc,GAA4B;QAC9C,UAAU,EAAE,QAAQ;QACpB,WAAW,EAAE,CAAC,UAAU,CAAC;QACzB,UAAU,EAAE,GAAG;QACf,iBAAiB,EAAE,IAAI;QACvB,eAAe,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE;KACzC,CAAC;IACF,IAAI,KAAK,EAAE,CAAC;QACV,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC;QAC7B,cAAc,CAAC,MAAM,GAAG,MAAM,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,aAAa,UAAU,gBAAgB,kBAAkB,OAAO,CAAC,GAAG;QACxE,aAAa,EAAE,kBAAkB;QACjC,eAAe,EAAE,cAAc;QAC/B,UAAU,EAAE,SAAS;QACrB,WAAW,EAAE,UAAU;QACvB,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,GAAG,EAAE,iBAAiB;QACtB,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,cAAc;QACpB,cAAc,EAAE,cAAc;QAC9B,MAAM,EAAE,QAAQ;QAChB,cAAc,EAAE,UAAU;QAC1B,aAAa,EAAE,EAAE;QACjB,eAAe,EAAE,cAAc;QAC/B,gBAAgB,EAAE,EAAE,YAAY,EAAE,EAAE,cAAc,EAAE,aAAa,EAAE,EAAE;QACrE,kBAAkB,EAAE,WAAW;QAC/B,oBAAoB,EAAE,iBAAiB;QACvC,QAAQ,EAAE;YACR,aAAa,EAAE,sBAAsB;YACrC,IAAI,EAAE,QAAQ;YACd,KAAK,EACH,4LAA4L;SAC/L;QACD,UAAU,EAAE,GAAG;QACf,UAAU,EAAE,GAAG;KAChB,CAAC;IAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CACZ;YACE,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,UAAU;YACvB,aAAa,EAAE,UAAU;YACzB,aAAa,EAAE,sBAAsB;YACrC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,YAAY,EAAE,WAAW;YACzB,oBAAoB,EAAE,kBAAkB;YACxC,cAAc,EAAE,aAAa;YAC7B,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;SAC1B,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACrD,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;IACxE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CACZ;QACE,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,UAAU;QACzB,aAAa,EAAE,sBAAsB;QACrC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY,EAAE,WAAW;QACzB,oBAAoB,EAAE,kBAAkB;QACxC,cAAc,EAAE,aAAa;QAC7B,KAAK,EAAE,OAAO;KACf,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IACF,OAAO,CAAC,KAAK,CAAC;6DAC6C,sBAAsB;;;CAGlF,CAAC,CAAC;AACH,CAAC"}
|
|
@@ -24,6 +24,7 @@ export declare function runIngestCreate(params: {
|
|
|
24
24
|
no_deploy?: boolean;
|
|
25
25
|
/** Enable Iceberg storage flag on the source data product. */
|
|
26
26
|
iceberg?: boolean;
|
|
27
|
+
transport_mode?: string;
|
|
27
28
|
}, options?: IngestCmdOptions): Promise<void>;
|
|
28
29
|
/** @deprecated Use runIngestCreate — provision alias for 1–2 releases. */
|
|
29
30
|
export declare const runIngestProvision: typeof runIngestCreate;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-cmd.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ingest-cmd.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"ingest-cmd.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ingest-cmd.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAiBH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAgBD,wBAAsB,eAAe,CACnC,MAAM,EAAE;IACN,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,mEAAmE;IACnE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,qEAAqE;IACrE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,8DAA8D;IAC9D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,EACD,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,IAAI,CAAC,CA4Of;AAED,0EAA0E;AAC1E,eAAO,MAAM,kBAAkB,wBAAkB,CAAC"}
|
|
@@ -3,12 +3,13 @@
|
|
|
3
3
|
* Alias: `loxtep ingest provision` (deprecated).
|
|
4
4
|
* Default: local files only (no save_workflow_bundle / deploy). Use --deploy to push+deploy.
|
|
5
5
|
*/
|
|
6
|
-
import { existsSync, mkdirSync, writeFileSync } from
|
|
7
|
-
import { dirname, join } from
|
|
8
|
-
import { buildSdkIngestLocalPackage, validateSdkIngestPackageFiles, } from
|
|
9
|
-
import { findProjectDir } from
|
|
10
|
-
import { requireCliClient } from
|
|
11
|
-
import { formatLintResult, runLintCheck } from
|
|
6
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
7
|
+
import { dirname, join } from "node:path";
|
|
8
|
+
import { buildSdkIngestLocalPackage, validateSdkIngestPackageFiles, } from "../../lib/sdk-ingest-bundle.js";
|
|
9
|
+
import { findProjectDir } from "../project-context.js";
|
|
10
|
+
import { requireCliClient } from "../create-cli-client.js";
|
|
11
|
+
import { formatLintResult, runLintCheck } from "./lint-cmd.js";
|
|
12
|
+
import { parseEventWriterTransportMode, } from "../../client/flow-types.js";
|
|
12
13
|
function writePackageFiles(projectDir, files) {
|
|
13
14
|
const written = [];
|
|
14
15
|
for (const [relPath, entity] of Object.entries(files)) {
|
|
@@ -23,23 +24,32 @@ export async function runIngestCreate(params, options = {}) {
|
|
|
23
24
|
const cwd = options.cwd ?? process.cwd();
|
|
24
25
|
const projectDir = findProjectDir(cwd) ?? cwd;
|
|
25
26
|
const { client, config } = await requireCliClient(options);
|
|
26
|
-
const dataProductName = params.name ??
|
|
27
|
+
const dataProductName = params.name ?? "app-events";
|
|
28
|
+
let transportMode;
|
|
29
|
+
try {
|
|
30
|
+
transportMode = parseEventWriterTransportMode(params.transport_mode);
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
34
|
+
process.exitCode = 1;
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
27
37
|
const projectId = params.project_id ?? config.project_id;
|
|
28
38
|
const instanceId = (params.instance_id && params.instance_id.trim()) || config.instance_id;
|
|
29
39
|
if (!projectId) {
|
|
30
|
-
console.error(
|
|
40
|
+
console.error("Missing project_id. Run `loxtep init` and attach a project first.");
|
|
31
41
|
process.exitCode = 1;
|
|
32
42
|
return;
|
|
33
43
|
}
|
|
34
44
|
if (!instanceId) {
|
|
35
|
-
console.error(
|
|
45
|
+
console.error("Missing instance_id. Run `loxtep attach --instance <id>` or pass --instance-id.");
|
|
36
46
|
process.exitCode = 1;
|
|
37
47
|
return;
|
|
38
48
|
}
|
|
39
49
|
const user = await client.session.get_current_user();
|
|
40
50
|
const organizationId = user.organization_id ?? client.organization_id;
|
|
41
51
|
if (!organizationId) {
|
|
42
|
-
console.error(
|
|
52
|
+
console.error("Missing organization_id in session.");
|
|
43
53
|
process.exitCode = 1;
|
|
44
54
|
return;
|
|
45
55
|
}
|
|
@@ -48,7 +58,7 @@ export async function runIngestCreate(params, options = {}) {
|
|
|
48
58
|
const domains = await client.define.domains.list({ page_size: 20 });
|
|
49
59
|
const items = domains.items ?? [];
|
|
50
60
|
if (items.length === 0) {
|
|
51
|
-
console.error(
|
|
61
|
+
console.error("No domains found. Create one in the Web UI (Governance → Domains) or pass --domain-id.");
|
|
52
62
|
process.exitCode = 1;
|
|
53
63
|
return;
|
|
54
64
|
}
|
|
@@ -63,22 +73,22 @@ export async function runIngestCreate(params, options = {}) {
|
|
|
63
73
|
connectorReused = true;
|
|
64
74
|
}
|
|
65
75
|
else {
|
|
66
|
-
console.error(
|
|
76
|
+
console.error("Looking up existing SDK connectors…");
|
|
67
77
|
const listed = await client.connect.connectors.list({
|
|
68
|
-
connector_type:
|
|
78
|
+
connector_type: "sdk",
|
|
69
79
|
page_size: 50,
|
|
70
80
|
});
|
|
71
|
-
const existing = (listed.items ?? []).find(c => c.connector_type ===
|
|
81
|
+
const existing = (listed.items ?? []).find((c) => c.connector_type === "sdk");
|
|
72
82
|
if (existing) {
|
|
73
83
|
connector = existing;
|
|
74
84
|
connectorReused = true;
|
|
75
85
|
console.error(`Reusing SDK connector: ${connector.connector_id}`);
|
|
76
86
|
}
|
|
77
87
|
else {
|
|
78
|
-
console.error(
|
|
88
|
+
console.error("Creating SDK connector…");
|
|
79
89
|
const connectorMetadata = {
|
|
80
90
|
name: `${dataProductName} SDK`,
|
|
81
|
-
created_by:
|
|
91
|
+
created_by: "loxtep-ingest-create",
|
|
82
92
|
project_id: projectId,
|
|
83
93
|
instance_id: instanceId,
|
|
84
94
|
};
|
|
@@ -86,12 +96,12 @@ export async function runIngestCreate(params, options = {}) {
|
|
|
86
96
|
connectorMetadata.region = config.region;
|
|
87
97
|
}
|
|
88
98
|
connector = await client.connect.connectors.create({
|
|
89
|
-
connector_type:
|
|
99
|
+
connector_type: "sdk",
|
|
90
100
|
metadata: connectorMetadata,
|
|
91
101
|
});
|
|
92
102
|
}
|
|
93
103
|
}
|
|
94
|
-
const localConnectorPath = join(projectDir,
|
|
104
|
+
const localConnectorPath = join(projectDir, "connectors", `${connector.connector_id}.json`);
|
|
95
105
|
const includeConnectorFile = !existsSync(localConnectorPath);
|
|
96
106
|
const pkg = buildSdkIngestLocalPackage({
|
|
97
107
|
organization_id: organizationId,
|
|
@@ -104,10 +114,11 @@ export async function runIngestCreate(params, options = {}) {
|
|
|
104
114
|
connector,
|
|
105
115
|
include_connector_file: includeConnectorFile,
|
|
106
116
|
iceberg_enabled: params.iceberg === true,
|
|
117
|
+
transport_mode: transportMode,
|
|
107
118
|
});
|
|
108
119
|
const schemaErrors = validateSdkIngestPackageFiles(pkg.files);
|
|
109
120
|
if (schemaErrors.length > 0) {
|
|
110
|
-
console.error(
|
|
121
|
+
console.error("Generated package failed schema validation:");
|
|
111
122
|
for (const err of schemaErrors) {
|
|
112
123
|
console.error(` ${err.path}: ${err.message}`);
|
|
113
124
|
}
|
|
@@ -123,9 +134,10 @@ export async function runIngestCreate(params, options = {}) {
|
|
|
123
134
|
data_product_id: pkg.data_product_id,
|
|
124
135
|
data_product_name: pkg.data_product_name,
|
|
125
136
|
iceberg_enabled: params.iceberg === true,
|
|
137
|
+
transport_mode: transportMode,
|
|
126
138
|
files: Object.keys(pkg.files),
|
|
127
139
|
}, null, 2));
|
|
128
|
-
console.error(
|
|
140
|
+
console.error("Dry run OK — no files written. Re-run without --dry-run to write the local package.");
|
|
129
141
|
return;
|
|
130
142
|
}
|
|
131
143
|
const written = writePackageFiles(projectDir, pkg.files);
|
|
@@ -147,23 +159,23 @@ export async function runIngestCreate(params, options = {}) {
|
|
|
147
159
|
let reindexResult;
|
|
148
160
|
if (shouldDeploy) {
|
|
149
161
|
const flatFiles = {
|
|
150
|
-
|
|
162
|
+
"workflow.json": pkg.files[`workflows/${pkg.workflow_id}/workflow.json`],
|
|
151
163
|
[`connections/${pkg.connection_id}.json`]: pkg.files[`workflows/${pkg.workflow_id}/connections/${pkg.connection_id}.json`],
|
|
152
164
|
[`data-products/${pkg.data_product_id}.json`]: pkg.files[`workflows/${pkg.workflow_id}/data-products/${pkg.data_product_id}.json`],
|
|
153
165
|
};
|
|
154
|
-
console.error(
|
|
166
|
+
console.error("Saving workflow bundle…");
|
|
155
167
|
saveResult = await client.build.workflows.save_workflow_bundle(projectId, {
|
|
156
168
|
files: flatFiles,
|
|
157
169
|
dry_run: false,
|
|
158
170
|
});
|
|
159
|
-
console.error(
|
|
171
|
+
console.error("Reindexing project workspace…");
|
|
160
172
|
try {
|
|
161
173
|
reindexResult = await client.workspace.projects.reindex(projectId);
|
|
162
174
|
}
|
|
163
175
|
catch (err) {
|
|
164
176
|
console.error(`Warning: reindex failed (${err instanceof Error ? err.message : String(err)}).`);
|
|
165
177
|
}
|
|
166
|
-
console.error(
|
|
178
|
+
console.error("Deploying project to instance…");
|
|
167
179
|
deployResult = await client.build.workflows.deploy({
|
|
168
180
|
project_id: projectId,
|
|
169
181
|
instance_id: instanceId,
|
|
@@ -176,6 +188,7 @@ export async function runIngestCreate(params, options = {}) {
|
|
|
176
188
|
data_product_id: pkg.data_product_id,
|
|
177
189
|
data_product_name: pkg.data_product_name,
|
|
178
190
|
iceberg_enabled: params.iceberg === true,
|
|
191
|
+
transport_mode: transportMode,
|
|
179
192
|
files: written,
|
|
180
193
|
save: saveResult,
|
|
181
194
|
reindex: reindexResult,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-cmd.js","sourceRoot":"","sources":["../../../src/cli/commands/ingest-cmd.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EACL,0BAA0B,EAC1B,6BAA6B,GAC9B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"ingest-cmd.js","sourceRoot":"","sources":["../../../src/cli/commands/ingest-cmd.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EACL,0BAA0B,EAC1B,6BAA6B,GAC9B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC/D,OAAO,EACL,6BAA6B,GAE9B,MAAM,4BAA4B,CAAC;AAUpC,SAAS,iBAAiB,CACxB,UAAkB,EAClB,KAA8C;IAE9C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACvC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAeC,EACD,UAA4B,EAAE;IAE9B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;IAE9C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC;IACpD,IAAI,aAAuC,CAAC;IAC5C,IAAI,CAAC;QACH,aAAa,GAAG,6BAA6B,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IACvE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;IACzD,MAAM,UAAU,GACd,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC;IAE1E,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,mEAAmE,CACpE,CAAC;QACF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CACX,iFAAiF,CAClF,CAAC;QACF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;IACrD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,CAAC;IACtE,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACrD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,IAAI,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC;IAChC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CACX,wFAAwF,CACzF,CAAC;YACF,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QACD,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9B,OAAO,CAAC,KAAK,CAAC,iBAAiB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,GAAG,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,SAAoB,CAAC;IACzB,IAAI,eAAe,GAAG,KAAK,CAAC;IAE5B,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,OAAO,CAAC,KAAK,CAAC,mBAAmB,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC;QACzD,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACrE,eAAe,GAAG,IAAI,CAAC;IACzB,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;YAClD,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,EAAE;SACd,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CACxC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,KAAK,CAClC,CAAC;QACF,IAAI,QAAQ,EAAE,CAAC;YACb,SAAS,GAAG,QAAQ,CAAC;YACrB,eAAe,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,0BAA0B,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC;QACpE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACzC,MAAM,iBAAiB,GAA4B;gBACjD,IAAI,EAAE,GAAG,eAAe,MAAM;gBAC9B,UAAU,EAAE,sBAAsB;gBAClC,UAAU,EAAE,SAAS;gBACrB,WAAW,EAAE,UAAU;aACxB,CAAC;YACF,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,iBAAiB,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC3C,CAAC;YACD,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC;gBACjD,cAAc,EAAE,KAAK;gBACrB,QAAQ,EAAE,iBAAiB;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,kBAAkB,GAAG,IAAI,CAC7B,UAAU,EACV,YAAY,EACZ,GAAG,SAAS,CAAC,YAAY,OAAO,CACjC,CAAC;IACF,MAAM,oBAAoB,GAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;IAE7D,MAAM,GAAG,GAAG,0BAA0B,CAAC;QACrC,eAAe,EAAE,cAAc;QAC/B,UAAU,EAAE,SAAS;QACrB,SAAS,EAAE,QAAQ;QACnB,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,iBAAiB,EAAE,eAAe;QAClC,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,SAAS;QACT,sBAAsB,EAAE,oBAAoB;QAC5C,eAAe,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI;QACxC,cAAc,EAAE,aAAa;KAC9B,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,6BAA6B,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QAC7D,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CACZ;YACE,OAAO,EAAE,IAAI;YACb,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,gBAAgB,EAAE,eAAe;YACjC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,eAAe,EAAE,GAAG,CAAC,eAAe;YACpC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;YACxC,eAAe,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI;YACxC,cAAc,EAAE,aAAa;YAC7B,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;SAC9B,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,OAAO,CAAC,KAAK,CACX,qFAAqF,CACtF,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IACzD,OAAO,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,MAAM,iBAAiB,CAAC,CAAC;IACxD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;IAC7E,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC;IAEzE,IAAI,UAAmB,CAAC;IACxB,IAAI,YAAqB,CAAC;IAC1B,IAAI,aAAsB,CAAC;IAE3B,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,SAAS,GAA4C;YACzD,eAAe,EAAE,GAAG,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,WAAW,gBAAgB,CAAC;YACxE,CAAC,eAAe,GAAG,CAAC,aAAa,OAAO,CAAC,EACvC,GAAG,CAAC,KAAK,CACP,aAAa,GAAG,CAAC,WAAW,gBAAgB,GAAG,CAAC,aAAa,OAAO,CACrE;YACH,CAAC,iBAAiB,GAAG,CAAC,eAAe,OAAO,CAAC,EAC3C,GAAG,CAAC,KAAK,CACP,aAAa,GAAG,CAAC,WAAW,kBAAkB,GAAG,CAAC,eAAe,OAAO,CACzE;SACJ,CAAC;QAEF,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACzC,UAAU,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,SAAS,EAAE;YACxE,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAEH,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAC/C,IAAI,CAAC;YACH,aAAa,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CACX,4BAA4B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CACjF,CAAC;QACJ,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,YAAY,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC;YACjD,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,UAAU;SACxB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CACZ;QACE,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,gBAAgB,EAAE,eAAe;QACjC,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,eAAe,EAAE,GAAG,CAAC,eAAe;QACpC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;QACxC,eAAe,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI;QACxC,cAAc,EAAE,aAAa;QAC7B,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,UAAU;QAChB,OAAO,EAAE,aAAa;QACtB,MAAM,EAAE,YAAY;KACrB,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IAEF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC;2BACS,eAAe;;kBAExB,eAAe;CAChC,CAAC,CAAC;IACD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC;WACP,eAAe,sBAAsB,eAAe;CAC9D,CAAC,CAAC;IACD,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transform-cmd.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/transform-cmd.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"transform-cmd.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/transform-cmd.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,MAAM,WAAW,mBAAmB;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAkBD;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE;IACN,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,EACD,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC,CA+Jf"}
|