@loxtep/sdk 0.11.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/cli/commands/attach-cmd.d.ts +1 -1
- package/dist/cli/commands/attach-cmd.js +1 -1
- package/dist/cli/commands/delivery-cmd.d.ts +1 -0
- package/dist/cli/commands/delivery-cmd.d.ts.map +1 -1
- package/dist/cli/commands/delivery-cmd.js +60 -44
- package/dist/cli/commands/delivery-cmd.js.map +1 -1
- package/dist/cli/commands/ingest-cmd.d.ts +1 -0
- package/dist/cli/commands/ingest-cmd.d.ts.map +1 -1
- package/dist/cli/commands/ingest-cmd.js +37 -24
- package/dist/cli/commands/ingest-cmd.js.map +1 -1
- package/dist/cli/commands/start-cmd.d.ts +25 -0
- package/dist/cli/commands/start-cmd.d.ts.map +1 -0
- package/dist/cli/commands/start-cmd.js +145 -0
- package/dist/cli/commands/start-cmd.js.map +1 -0
- package/dist/cli/commands/transform-cmd.d.ts +1 -0
- package/dist/cli/commands/transform-cmd.d.ts.map +1 -1
- package/dist/cli/commands/transform-cmd.js +36 -23
- package/dist/cli/commands/transform-cmd.js.map +1 -1
- package/dist/cli/help.d.ts.map +1 -1
- package/dist/cli/help.js +9 -3
- package/dist/cli/help.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +576 -515
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/setup-prompt.d.ts +2 -0
- package/dist/cli/setup-prompt.d.ts.map +1 -0
- package/dist/cli/setup-prompt.js +3 -0
- package/dist/cli/setup-prompt.js.map +1 -0
- package/dist/client/data-products.d.ts +10 -8
- package/dist/client/data-products.d.ts.map +1 -1
- package/dist/client/data-products.js +142 -93
- package/dist/client/data-products.js.map +1 -1
- package/dist/client/flow-types.d.ts +16 -6
- package/dist/client/flow-types.d.ts.map +1 -1
- package/dist/client/flow-types.js +6 -1
- package/dist/client/flow-types.js.map +1 -1
- package/dist/client/index.d.ts +89 -88
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +13 -12
- package/dist/client/index.js.map +1 -1
- package/dist/client/projects-types.d.ts +1 -1
- package/dist/client/queues.d.ts +5 -3
- package/dist/client/queues.d.ts.map +1 -1
- package/dist/client/queues.js +26 -21
- package/dist/client/queues.js.map +1 -1
- package/dist/client/workflows.d.ts +7 -7
- package/dist/client/workflows.d.ts.map +1 -1
- package/dist/client/workflows.js +71 -63
- package/dist/client/workflows.js.map +1 -1
- package/dist/lib/sdk-ingest-bundle.d.ts +5 -1
- package/dist/lib/sdk-ingest-bundle.d.ts.map +1 -1
- package/dist/lib/sdk-ingest-bundle.js +38 -29
- package/dist/lib/sdk-ingest-bundle.js.map +1 -1
- package/dist/rstreams/event-bridge.d.ts +4 -4
- package/dist/rstreams/event-bridge.d.ts.map +1 -1
- package/dist/rstreams/event-bridge.js +23 -16
- package/dist/rstreams/event-bridge.js.map +1 -1
- package/docs/event-writer-transport-modes.md +47 -0
- package/docs/getting-started.md +19 -0
- package/docs/sdk-first-ingest.md +14 -12
- package/docs/sdk-mcp-mapping.md +8 -3
- package/package.json +1 -1
- package/schemas/entity-json-schemas/SCHEMA_SYNC.md +1 -1
- package/schemas/entity-json-schemas/SOURCE.json +5 -4
- package/schemas/entity-json-schemas/connection.json +35 -0
- package/schemas/entity-json-schemas/data-product.json +16 -0
- package/schemas/entity-json-schemas/grant.json +146 -0
- package/schemas/entity-json-schemas/transformation.json +19 -0
|
@@ -3,40 +3,42 @@
|
|
|
3
3
|
* get_writer, get_reader, invalidate_cache.
|
|
4
4
|
* Customer-facing surface: data_products.
|
|
5
5
|
*/
|
|
6
|
-
import { WarehouseQueryError } from
|
|
7
|
-
import { readQueueBatch, createQueueWriter, } from
|
|
8
|
-
import { NotFoundError } from
|
|
9
|
-
import { AuthorizationError } from
|
|
10
|
-
import { StreamingError } from
|
|
11
|
-
import { resolveStreamsConfiguration } from
|
|
12
|
-
import { createRStreamsSdk } from
|
|
6
|
+
import { WarehouseQueryError } from "./data-products-types.js";
|
|
7
|
+
import { readQueueBatch, createQueueWriter, } from "../rstreams/event-bridge.js";
|
|
8
|
+
import { NotFoundError } from "../errors/resource.js";
|
|
9
|
+
import { AuthorizationError } from "../errors/auth.js";
|
|
10
|
+
import { StreamingError } from "../errors/streaming.js";
|
|
11
|
+
import { resolveStreamsConfiguration } from "../rstreams/configuration.js";
|
|
12
|
+
import { createRStreamsSdk } from "../rstreams/leo-runtime.js";
|
|
13
13
|
function buildQueryString(params) {
|
|
14
14
|
const search = new URLSearchParams();
|
|
15
15
|
for (const [k, v] of Object.entries(params)) {
|
|
16
16
|
if (v === undefined || v === null)
|
|
17
17
|
continue;
|
|
18
18
|
if (Array.isArray(v)) {
|
|
19
|
-
search.set(k, v.join(
|
|
19
|
+
search.set(k, v.join(","));
|
|
20
20
|
}
|
|
21
21
|
else {
|
|
22
22
|
search.set(k, String(v));
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
25
|
const qs = search.toString();
|
|
26
|
-
return qs ? `?${qs}` :
|
|
26
|
+
return qs ? `?${qs}` : "";
|
|
27
27
|
}
|
|
28
28
|
function queueNameFromDeploymentBindings(asset) {
|
|
29
29
|
const raw = asset.deployment_bindings?.queue_name;
|
|
30
|
-
if (typeof raw !==
|
|
30
|
+
if (typeof raw !== "string")
|
|
31
31
|
return undefined;
|
|
32
32
|
const trimmed = raw.trim();
|
|
33
33
|
return trimmed.length > 0 ? trimmed : undefined;
|
|
34
34
|
}
|
|
35
35
|
function unwrapRecord(res) {
|
|
36
|
-
if (!res || typeof res !==
|
|
36
|
+
if (!res || typeof res !== "object" || Array.isArray(res))
|
|
37
37
|
return {};
|
|
38
38
|
const record = res;
|
|
39
|
-
if (record.data &&
|
|
39
|
+
if (record.data &&
|
|
40
|
+
typeof record.data === "object" &&
|
|
41
|
+
!Array.isArray(record.data)) {
|
|
40
42
|
return record.data;
|
|
41
43
|
}
|
|
42
44
|
return record;
|
|
@@ -51,8 +53,8 @@ function eventIdentity(event) {
|
|
|
51
53
|
if (!event)
|
|
52
54
|
return null;
|
|
53
55
|
const record = event;
|
|
54
|
-
for (const key of [
|
|
55
|
-
if (typeof record[key] ===
|
|
56
|
+
for (const key of ["eid", "event_id", "id"]) {
|
|
57
|
+
if (typeof record[key] === "string" && record[key])
|
|
56
58
|
return record[key];
|
|
57
59
|
}
|
|
58
60
|
return null;
|
|
@@ -70,7 +72,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
70
72
|
include_lineage: options.include_lineage,
|
|
71
73
|
include_contracts: options.include_contracts,
|
|
72
74
|
})
|
|
73
|
-
:
|
|
75
|
+
: "";
|
|
74
76
|
const res = await http.get(`/dataproducts/${encodeURIComponent(id)}${qs}`);
|
|
75
77
|
return res.data;
|
|
76
78
|
},
|
|
@@ -80,11 +82,12 @@ export function createDataProductsApi(http, deps) {
|
|
|
80
82
|
*/
|
|
81
83
|
async get_lexicon(id) {
|
|
82
84
|
const asset = await this.get(id);
|
|
83
|
-
const glossary_terms = asset
|
|
84
|
-
.
|
|
85
|
-
|
|
85
|
+
const glossary_terms = asset.glossary_terms ??
|
|
86
|
+
asset.metadata
|
|
87
|
+
?.business_glossary ??
|
|
86
88
|
{};
|
|
87
|
-
const field_glossary_map = asset.metadata
|
|
89
|
+
const field_glossary_map = asset.metadata
|
|
90
|
+
?.field_glossary_map ??
|
|
88
91
|
undefined;
|
|
89
92
|
return { glossary_terms, field_glossary_map };
|
|
90
93
|
},
|
|
@@ -92,8 +95,8 @@ export function createDataProductsApi(http, deps) {
|
|
|
92
95
|
const params = {
|
|
93
96
|
page: filters?.page ?? 1,
|
|
94
97
|
page_size: filters?.page_size ?? 100,
|
|
95
|
-
sort_by: filters?.sort_by ??
|
|
96
|
-
sort_order: filters?.sort_order ??
|
|
98
|
+
sort_by: filters?.sort_by ?? "created_at",
|
|
99
|
+
sort_order: filters?.sort_order ?? "desc",
|
|
97
100
|
};
|
|
98
101
|
if (filters?.domain_id)
|
|
99
102
|
params.domain_id = filters.domain_id;
|
|
@@ -108,7 +111,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
108
111
|
if (filters?.search)
|
|
109
112
|
params.search = filters.search;
|
|
110
113
|
if (filters?.tags?.length)
|
|
111
|
-
params.tags = filters.tags.join(
|
|
114
|
+
params.tags = filters.tags.join(",");
|
|
112
115
|
const qs = buildQueryString(params);
|
|
113
116
|
const res = await http.get(`/dataproducts${qs}`);
|
|
114
117
|
return res.data;
|
|
@@ -116,7 +119,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
116
119
|
async search(query, filters) {
|
|
117
120
|
const params = {
|
|
118
121
|
q: query,
|
|
119
|
-
type: filters?.type ??
|
|
122
|
+
type: filters?.type ?? "data_product",
|
|
120
123
|
limit: filters?.limit ?? 20,
|
|
121
124
|
offset: filters?.offset ?? 0,
|
|
122
125
|
};
|
|
@@ -124,45 +127,54 @@ export function createDataProductsApi(http, deps) {
|
|
|
124
127
|
return http.get(`/search${qs}`);
|
|
125
128
|
},
|
|
126
129
|
async query_receipt(id, sql) {
|
|
127
|
-
const res = await http.post(
|
|
130
|
+
const res = await http.post("/dataproducts/warehouse/execute", {
|
|
128
131
|
sql,
|
|
129
132
|
data_product_ids_hint: [id],
|
|
130
133
|
});
|
|
131
134
|
const wrapped = res.data;
|
|
132
|
-
const body = wrapped && typeof wrapped ===
|
|
135
|
+
const body = wrapped && typeof wrapped === "object" && !Array.isArray(wrapped)
|
|
133
136
|
? wrapped
|
|
134
137
|
: res;
|
|
135
|
-
const status = typeof body.status ===
|
|
136
|
-
|
|
138
|
+
const status = typeof body.status === "string" && body.status.length > 0
|
|
139
|
+
? body.status
|
|
140
|
+
: undefined;
|
|
141
|
+
const rows = Array.isArray(body.rows)
|
|
142
|
+
? body.rows
|
|
143
|
+
: [];
|
|
137
144
|
const columns = Array.isArray(body.columns)
|
|
138
|
-
? body.columns.map(column => String(column))
|
|
145
|
+
? body.columns.map((column) => String(column))
|
|
139
146
|
: undefined;
|
|
140
147
|
return {
|
|
141
148
|
data_product_id: id,
|
|
142
|
-
query_id: typeof body.query_id ===
|
|
149
|
+
query_id: typeof body.query_id === "string" ? body.query_id : undefined,
|
|
143
150
|
status,
|
|
144
151
|
columns,
|
|
145
152
|
rows,
|
|
146
|
-
row_count: typeof body.row_count ===
|
|
147
|
-
execution_time_ms: typeof body.execution_time_ms ===
|
|
148
|
-
|
|
149
|
-
|
|
153
|
+
row_count: typeof body.row_count === "number" ? body.row_count : rows.length,
|
|
154
|
+
execution_time_ms: typeof body.execution_time_ms === "number"
|
|
155
|
+
? body.execution_time_ms
|
|
156
|
+
: undefined,
|
|
157
|
+
error: typeof body.error === "string" ? body.error : undefined,
|
|
158
|
+
error_code: typeof body.error_code === "string" ? body.error_code : undefined,
|
|
150
159
|
tables_referenced: body.tables_referenced,
|
|
151
160
|
recovery: body.recovery,
|
|
152
|
-
iceberg_health_status: typeof body.iceberg_health_status ===
|
|
153
|
-
|
|
161
|
+
iceberg_health_status: typeof body.iceberg_health_status === "string"
|
|
162
|
+
? body.iceberg_health_status
|
|
163
|
+
: undefined,
|
|
164
|
+
completion_evidence: status ? "native" : "missing",
|
|
154
165
|
};
|
|
155
166
|
},
|
|
156
167
|
async query(id, sql) {
|
|
157
168
|
const receipt = await this.query_receipt(id, sql);
|
|
158
|
-
if (receipt.completion_evidence !==
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
169
|
+
if (receipt.completion_evidence !== "native" ||
|
|
170
|
+
receipt.status !== "completed") {
|
|
171
|
+
const code = receipt.status === "failed"
|
|
172
|
+
? receipt.error_code || "query_failed"
|
|
173
|
+
: "missing_completion";
|
|
162
174
|
throw new WarehouseQueryError(receipt.error ||
|
|
163
175
|
(receipt.status
|
|
164
176
|
? `Warehouse query status is ${receipt.status}`
|
|
165
|
-
:
|
|
177
|
+
: "Warehouse query is missing completion evidence"), { query_id: receipt.query_id, status: receipt.status, code });
|
|
166
178
|
}
|
|
167
179
|
const rows = receipt.rows;
|
|
168
180
|
return {
|
|
@@ -176,27 +188,27 @@ export function createDataProductsApi(http, deps) {
|
|
|
176
188
|
};
|
|
177
189
|
},
|
|
178
190
|
async export_events(id, options = {}) {
|
|
179
|
-
const instanceId = http.getDefaultHeader?.(
|
|
191
|
+
const instanceId = http.getDefaultHeader?.("x-loxtep-instance-id");
|
|
180
192
|
if (!instanceId) {
|
|
181
|
-
const err = new Error(
|
|
182
|
-
err.code =
|
|
193
|
+
const err = new Error("instance_required: export needs --instance or an attached project instance_id");
|
|
194
|
+
err.code = "instance_required";
|
|
183
195
|
throw err;
|
|
184
196
|
}
|
|
185
197
|
if (!options.from_beginning && !options.start) {
|
|
186
|
-
const err = new Error(
|
|
187
|
-
err.code =
|
|
198
|
+
const err = new Error("export requires start or from_beginning");
|
|
199
|
+
err.code = "start_required";
|
|
188
200
|
throw err;
|
|
189
201
|
}
|
|
190
202
|
const asset = await this.get(id);
|
|
191
203
|
const queueName = queueNameFromDeploymentBindings(asset);
|
|
192
204
|
if (!queueName) {
|
|
193
205
|
const err = new Error(`Data product ${id} has no deployment_bindings.queue_name; deploy the workflow first`);
|
|
194
|
-
err.code =
|
|
206
|
+
err.code = "no_queue";
|
|
195
207
|
throw err;
|
|
196
208
|
}
|
|
197
209
|
const limit = options.limit ?? 100;
|
|
198
210
|
const maxPages = options.max_pages ?? 1;
|
|
199
|
-
const startSent = options.from_beginning ?
|
|
211
|
+
const startSent = options.from_beginning ? "0" : String(options.start);
|
|
200
212
|
const events = [];
|
|
201
213
|
let start = startSent;
|
|
202
214
|
let pages = 0;
|
|
@@ -204,12 +216,12 @@ export function createDataProductsApi(http, deps) {
|
|
|
204
216
|
let truncation = null;
|
|
205
217
|
let serverCursor = null;
|
|
206
218
|
let nextStart = null;
|
|
207
|
-
let nextStartSemantics =
|
|
219
|
+
let nextStartSemantics = "The trace API does not document a next cursor. This page did not fill the requested limit.";
|
|
208
220
|
while (pages < maxPages && events.length < limit && start !== undefined) {
|
|
209
221
|
pages += 1;
|
|
210
222
|
const qs = buildQueryString({ start, end: options.end });
|
|
211
223
|
const res = await http.get(`/observe/trace/${encodeURIComponent(queueName)}/events${qs}`);
|
|
212
|
-
const payload = res.data && typeof res.data ===
|
|
224
|
+
const payload = res.data && typeof res.data === "object" && !Array.isArray(res.data)
|
|
213
225
|
? res.data
|
|
214
226
|
: res;
|
|
215
227
|
const batch = Array.isArray(payload.events)
|
|
@@ -218,38 +230,41 @@ export function createDataProductsApi(http, deps) {
|
|
|
218
230
|
? res.events
|
|
219
231
|
: [];
|
|
220
232
|
const cursorRaw = payload.next ?? payload.next_start ?? payload.next_cursor ?? res.next;
|
|
221
|
-
const cursor = typeof cursorRaw ===
|
|
233
|
+
const cursor = typeof cursorRaw === "string" && cursorRaw.length > 0
|
|
234
|
+
? cursorRaw
|
|
235
|
+
: null;
|
|
222
236
|
serverCursor = cursor;
|
|
223
237
|
const room = limit - events.length;
|
|
224
238
|
const slice = batch.slice(0, room);
|
|
225
239
|
events.push(...slice);
|
|
226
240
|
if (slice.length < batch.length) {
|
|
227
241
|
truncated = true;
|
|
228
|
-
truncation =
|
|
242
|
+
truncation = "client_limit";
|
|
229
243
|
nextStart = eventIdentity(slice[slice.length - 1]);
|
|
230
244
|
nextStartSemantics =
|
|
231
|
-
|
|
245
|
+
"client_limit: the server did not apply this limit. next_start is the last included event id. Trace start is inclusive, so the next page may repeat that event.";
|
|
232
246
|
break;
|
|
233
247
|
}
|
|
234
248
|
if (events.length >= limit) {
|
|
235
249
|
truncated = true;
|
|
236
|
-
truncation =
|
|
250
|
+
truncation =
|
|
251
|
+
pages >= maxPages && cursor ? "max_pages" : "client_limit";
|
|
237
252
|
nextStart = cursor ?? eventIdentity(events[events.length - 1]);
|
|
238
253
|
nextStartSemantics = cursor
|
|
239
|
-
?
|
|
240
|
-
:
|
|
254
|
+
? "server next cursor"
|
|
255
|
+
: "client_limit: the server did not provide a next cursor. next_start is the last included event id and may overlap if start is inclusive.";
|
|
241
256
|
break;
|
|
242
257
|
}
|
|
243
258
|
if (!cursor) {
|
|
244
259
|
nextStartSemantics =
|
|
245
|
-
|
|
260
|
+
"The server did not provide a next cursor. This response is the page that was returned.";
|
|
246
261
|
break;
|
|
247
262
|
}
|
|
248
263
|
if (pages >= maxPages) {
|
|
249
264
|
truncated = true;
|
|
250
|
-
truncation =
|
|
265
|
+
truncation = "max_pages";
|
|
251
266
|
nextStart = cursor;
|
|
252
|
-
nextStartSemantics =
|
|
267
|
+
nextStartSemantics = "server next cursor; max_pages stopped the loop";
|
|
253
268
|
break;
|
|
254
269
|
}
|
|
255
270
|
start = cursor;
|
|
@@ -307,27 +322,27 @@ export function createDataProductsApi(http, deps) {
|
|
|
307
322
|
},
|
|
308
323
|
async list_tables(id) {
|
|
309
324
|
// JWT path: warehouse tables list, filtered to this data product.
|
|
310
|
-
const res = await http.get(
|
|
325
|
+
const res = await http.get("/dataproducts/warehouse/tables");
|
|
311
326
|
const payload = res.data ?? res;
|
|
312
327
|
const tables = payload.tables ?? [];
|
|
313
328
|
const items = tables
|
|
314
|
-
.filter(t => !t.data_product_id || t.data_product_id === id)
|
|
315
|
-
.map(t => ({
|
|
316
|
-
name: String(t.sql_name ?? t.name ??
|
|
317
|
-
schema: typeof t.medallion ===
|
|
329
|
+
.filter((t) => !t.data_product_id || t.data_product_id === id)
|
|
330
|
+
.map((t) => ({
|
|
331
|
+
name: String(t.sql_name ?? t.name ?? ""),
|
|
332
|
+
schema: typeof t.medallion === "string" ? t.medallion : undefined,
|
|
318
333
|
...t,
|
|
319
334
|
}));
|
|
320
335
|
return { items };
|
|
321
336
|
},
|
|
322
337
|
async get_queue_info(id) {
|
|
323
338
|
if (!deps) {
|
|
324
|
-
throw new Error(
|
|
339
|
+
throw new Error("data_products.get_queue_info requires queues API; use LoxtepClient which wires it");
|
|
325
340
|
}
|
|
326
341
|
const asset = await this.get(id);
|
|
327
342
|
const queue_name = queueNameFromDeploymentBindings(asset);
|
|
328
343
|
if (!queue_name) {
|
|
329
344
|
return {
|
|
330
|
-
queue_name:
|
|
345
|
+
queue_name: "",
|
|
331
346
|
checkpoints: [],
|
|
332
347
|
readers: [],
|
|
333
348
|
writers: [],
|
|
@@ -338,7 +353,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
338
353
|
},
|
|
339
354
|
async get_reader_checkpoint(id, bot_id) {
|
|
340
355
|
if (!deps) {
|
|
341
|
-
throw new Error(
|
|
356
|
+
throw new Error("data_products.get_reader_checkpoint requires queues API; use LoxtepClient which wires it");
|
|
342
357
|
}
|
|
343
358
|
const asset = await this.get(id);
|
|
344
359
|
const queue_name = queueNameFromDeploymentBindings(asset);
|
|
@@ -348,10 +363,10 @@ export function createDataProductsApi(http, deps) {
|
|
|
348
363
|
return deps.get_reader_checkpoint(queue_name, bot_id);
|
|
349
364
|
},
|
|
350
365
|
async create(body) {
|
|
351
|
-
const res = await http.post(
|
|
366
|
+
const res = await http.post("/dataproducts", body);
|
|
352
367
|
const data = res.data;
|
|
353
368
|
if (!data)
|
|
354
|
-
throw new Error(
|
|
369
|
+
throw new Error("Invalid create data product response");
|
|
355
370
|
return data;
|
|
356
371
|
},
|
|
357
372
|
/**
|
|
@@ -360,7 +375,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
360
375
|
* Scoped to the caller's organization.
|
|
361
376
|
*/
|
|
362
377
|
async get_usage_map() {
|
|
363
|
-
const res = await http.get(
|
|
378
|
+
const res = await http.get("/dataproducts/usage-map");
|
|
364
379
|
const payload = res.data ?? res;
|
|
365
380
|
return {
|
|
366
381
|
nodes: payload.nodes ?? [],
|
|
@@ -374,8 +389,11 @@ export function createDataProductsApi(http, deps) {
|
|
|
374
389
|
}
|
|
375
390
|
catch (err) {
|
|
376
391
|
// Wrap 404s in NotFoundError for data product lookup failures
|
|
377
|
-
if (err &&
|
|
378
|
-
|
|
392
|
+
if (err &&
|
|
393
|
+
typeof err === "object" &&
|
|
394
|
+
"status_code" in err &&
|
|
395
|
+
err.status_code === 404) {
|
|
396
|
+
throw new NotFoundError(`Data product '${id}' not found`, "data_product", id);
|
|
379
397
|
}
|
|
380
398
|
throw err;
|
|
381
399
|
}
|
|
@@ -392,11 +410,11 @@ export function createDataProductsApi(http, deps) {
|
|
|
392
410
|
const bot_id = options?.bot_id;
|
|
393
411
|
const rsdk = deps?.rsdk ?? (await deps?.get_rsdk?.());
|
|
394
412
|
if (rsdk && !bot_id) {
|
|
395
|
-
throw new StreamingError(
|
|
413
|
+
throw new StreamingError("data_products.stream: bot_id is required when the stream bus is configured (LoxtepClient with streams/LEO_*). Use HTTP observe only by leaving bus env unset, or pass options.bot_id.", {
|
|
396
414
|
details: {
|
|
397
415
|
data_product_id: id,
|
|
398
416
|
queue_name,
|
|
399
|
-
hint:
|
|
417
|
+
hint: "Pass options.bot_id to stream(), or leave stream bus unconfigured to use HTTP observe.",
|
|
400
418
|
},
|
|
401
419
|
});
|
|
402
420
|
}
|
|
@@ -411,7 +429,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
411
429
|
// Map stream bus errors to typed SDK errors
|
|
412
430
|
const msg = err instanceof Error ? err.message : String(err);
|
|
413
431
|
if (/not found|does not exist|no such queue/i.test(msg)) {
|
|
414
|
-
throw new NotFoundError(`Queue '${queue_name}' not found. Verify the queue exists in your data product`,
|
|
432
|
+
throw new NotFoundError(`Queue '${queue_name}' not found. Verify the queue exists in your data product`, "queue", queue_name, { details: { data_product_id: id, bot_id } });
|
|
415
433
|
}
|
|
416
434
|
if (/unauthorized|forbidden|permission|access denied/i.test(msg)) {
|
|
417
435
|
throw new AuthorizationError(`Bot '${bot_id}' does not have read permission on queue '${queue_name}'`, { data_product_id: id, bot_id, queue_name });
|
|
@@ -447,7 +465,8 @@ export function createDataProductsApi(http, deps) {
|
|
|
447
465
|
break;
|
|
448
466
|
const last = events[events.length - 1];
|
|
449
467
|
httpStart =
|
|
450
|
-
last.event_id ??
|
|
468
|
+
last.event_id ??
|
|
469
|
+
last.id;
|
|
451
470
|
if (!httpStart)
|
|
452
471
|
break;
|
|
453
472
|
}
|
|
@@ -459,8 +478,11 @@ export function createDataProductsApi(http, deps) {
|
|
|
459
478
|
}
|
|
460
479
|
catch (err) {
|
|
461
480
|
// Wrap 404s in NotFoundError for data product lookup failures
|
|
462
|
-
if (err &&
|
|
463
|
-
|
|
481
|
+
if (err &&
|
|
482
|
+
typeof err === "object" &&
|
|
483
|
+
"status_code" in err &&
|
|
484
|
+
err.status_code === 404) {
|
|
485
|
+
throw new NotFoundError(`Data product '${id}' not found`, "data_product", id);
|
|
464
486
|
}
|
|
465
487
|
throw err;
|
|
466
488
|
}
|
|
@@ -475,8 +497,11 @@ export function createDataProductsApi(http, deps) {
|
|
|
475
497
|
}
|
|
476
498
|
const limit = options?.limit ?? 1000;
|
|
477
499
|
const start = options?.from_beginning
|
|
478
|
-
?
|
|
479
|
-
: (options?.from_eid ??
|
|
500
|
+
? "0"
|
|
501
|
+
: (options?.from_eid ??
|
|
502
|
+
options?.from_timestamp ??
|
|
503
|
+
options?.checkpoint_id ??
|
|
504
|
+
undefined);
|
|
480
505
|
const end = options?.to_eid ?? options?.to_timestamp ?? undefined;
|
|
481
506
|
const path = `/observe/trace/${encodeURIComponent(queue_name)}/events`;
|
|
482
507
|
const qs = buildQueryString({
|
|
@@ -502,12 +527,21 @@ export function createDataProductsApi(http, deps) {
|
|
|
502
527
|
async get_writer(idOrName, options) {
|
|
503
528
|
const resolver = deps?.resolver;
|
|
504
529
|
if (!resolver) {
|
|
505
|
-
throw new StreamingError(
|
|
530
|
+
throw new StreamingError("data_products.get_writer requires DataProductResolver. Use LoxtepClient which wires it automatically.", {
|
|
531
|
+
details: {
|
|
532
|
+
hint: "Instantiate via new LoxtepClient({ ... }) rather than calling createDataProductsApi directly.",
|
|
533
|
+
},
|
|
534
|
+
});
|
|
506
535
|
}
|
|
507
536
|
const { dataProduct, streamConfig } = await resolver.resolve(idOrName);
|
|
508
537
|
const botId = options?.bot_id ?? dataProduct.bot_id;
|
|
509
538
|
const queueName = dataProduct.queue_name;
|
|
510
|
-
const closedError = () => new StreamingError(
|
|
539
|
+
const closedError = () => new StreamingError("Cannot write to a closed FlowWriter. Create a new writer via data_products.get_writer().", {
|
|
540
|
+
details: {
|
|
541
|
+
data_product_id: dataProduct.data_product_id,
|
|
542
|
+
queue_name: queueName,
|
|
543
|
+
},
|
|
544
|
+
});
|
|
511
545
|
// Event publish is streams-only (Kinesis via stream config / BusWriter STS).
|
|
512
546
|
// There is no HTTP observe-proxy write path — configure LEO_* / instance
|
|
513
547
|
// stream environment in the account that owns the bus (including self-hosted).
|
|
@@ -517,7 +551,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
517
551
|
details: {
|
|
518
552
|
data_product_id: dataProduct.data_product_id,
|
|
519
553
|
instance_id: dataProduct.instance_id,
|
|
520
|
-
hint:
|
|
554
|
+
hint: "Configure instance stream bus credentials (LEO_* / streams config) in the AWS account that owns the bus. HTTP queue publish is not supported.",
|
|
521
555
|
},
|
|
522
556
|
});
|
|
523
557
|
}
|
|
@@ -530,13 +564,13 @@ export function createDataProductsApi(http, deps) {
|
|
|
530
564
|
throw new StreamingError(`Failed to resolve stream runtime for data product '${dataProduct.name}'.`, {
|
|
531
565
|
details: {
|
|
532
566
|
data_product_id: dataProduct.data_product_id,
|
|
533
|
-
hint:
|
|
567
|
+
hint: "Run `loxtep login` so STS BusWriter credentials are available, then retry.",
|
|
534
568
|
},
|
|
535
569
|
});
|
|
536
570
|
}
|
|
537
571
|
// The rstreams `load` stream (inside createQueueWriter) owns buffering, batching, backoff,
|
|
538
572
|
// and checkpointing — the wrapper just forwards business objects to it.
|
|
539
|
-
return createQueueWriter(rsdk, botId, queueName, closedError);
|
|
573
|
+
return createQueueWriter(rsdk, botId, queueName, closedError, options?.transport_mode);
|
|
540
574
|
},
|
|
541
575
|
/**
|
|
542
576
|
* Resolve a data product by name or UUID and return an AsyncIterable that reads
|
|
@@ -547,7 +581,11 @@ export function createDataProductsApi(http, deps) {
|
|
|
547
581
|
async get_reader(idOrName, options) {
|
|
548
582
|
const resolver = deps?.resolver;
|
|
549
583
|
if (!resolver) {
|
|
550
|
-
throw new StreamingError(
|
|
584
|
+
throw new StreamingError("data_products.get_reader requires DataProductResolver. Use LoxtepClient which wires it automatically.", {
|
|
585
|
+
details: {
|
|
586
|
+
hint: "Instantiate via new LoxtepClient({ ... }) rather than calling createDataProductsApi directly.",
|
|
587
|
+
},
|
|
588
|
+
});
|
|
551
589
|
}
|
|
552
590
|
const { dataProduct, streamConfig } = await resolver.resolve(idOrName);
|
|
553
591
|
const streamResources = resolveStreamsConfiguration(streamConfig);
|
|
@@ -556,7 +594,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
556
594
|
details: {
|
|
557
595
|
data_product_id: dataProduct.data_product_id,
|
|
558
596
|
instance_id: dataProduct.instance_id,
|
|
559
|
-
hint:
|
|
597
|
+
hint: "Verify the instance has a fully provisioned stream bus.",
|
|
560
598
|
},
|
|
561
599
|
});
|
|
562
600
|
}
|
|
@@ -567,7 +605,7 @@ export function createDataProductsApi(http, deps) {
|
|
|
567
605
|
throw new StreamingError(`Failed to resolve stream runtime for data product '${dataProduct.name}'.`, {
|
|
568
606
|
details: {
|
|
569
607
|
data_product_id: dataProduct.data_product_id,
|
|
570
|
-
hint:
|
|
608
|
+
hint: "Run `loxtep login` so STS BusWriter credentials are available, then retry.",
|
|
571
609
|
},
|
|
572
610
|
});
|
|
573
611
|
}
|
|
@@ -585,10 +623,19 @@ export function createDataProductsApi(http, deps) {
|
|
|
585
623
|
catch (err) {
|
|
586
624
|
const msg = err instanceof Error ? err.message : String(err);
|
|
587
625
|
if (/not found|does not exist|no such queue/i.test(msg)) {
|
|
588
|
-
throw new NotFoundError(`Queue '${queueName}' not found. Verify the data product has been deployed and the queue exists.`,
|
|
626
|
+
throw new NotFoundError(`Queue '${queueName}' not found. Verify the data product has been deployed and the queue exists.`, "queue", queueName, {
|
|
627
|
+
details: {
|
|
628
|
+
data_product_id: dataProduct.data_product_id,
|
|
629
|
+
bot_id: botId,
|
|
630
|
+
},
|
|
631
|
+
});
|
|
589
632
|
}
|
|
590
633
|
if (/unauthorized|forbidden|permission|access denied/i.test(msg)) {
|
|
591
|
-
throw new AuthorizationError(`Bot '${botId}' does not have read permission on queue '${queueName}'`, {
|
|
634
|
+
throw new AuthorizationError(`Bot '${botId}' does not have read permission on queue '${queueName}'`, {
|
|
635
|
+
data_product_id: dataProduct.data_product_id,
|
|
636
|
+
bot_id: botId,
|
|
637
|
+
queue_name: queueName,
|
|
638
|
+
});
|
|
592
639
|
}
|
|
593
640
|
throw err;
|
|
594
641
|
}
|
|
@@ -629,25 +676,27 @@ export function createDataProductsApi(http, deps) {
|
|
|
629
676
|
* Prerequisites are validated server-side. Returns 422 if not satisfied.
|
|
630
677
|
*/
|
|
631
678
|
async promote(data_product_id, target_tier) {
|
|
632
|
-
const res = await http.post(`/graph/promotions/${encodeURIComponent(data_product_id)}/promote`, {
|
|
679
|
+
const res = await http.post(`/graph/promotions/${encodeURIComponent(data_product_id)}/promote`, {
|
|
680
|
+
target_tier,
|
|
681
|
+
});
|
|
633
682
|
return res.data;
|
|
634
683
|
},
|
|
635
684
|
semantic_inference: {
|
|
636
685
|
async request(id) {
|
|
637
686
|
if (!id)
|
|
638
|
-
throw new Error(
|
|
687
|
+
throw new Error("id is required");
|
|
639
688
|
const res = await http.post(semanticInferencePath(id), {});
|
|
640
689
|
return unwrapData(res);
|
|
641
690
|
},
|
|
642
691
|
async get(id) {
|
|
643
692
|
if (!id)
|
|
644
|
-
throw new Error(
|
|
693
|
+
throw new Error("id is required");
|
|
645
694
|
const res = await http.get(semanticInferencePath(id));
|
|
646
695
|
return unwrapData(res);
|
|
647
696
|
},
|
|
648
697
|
async promote(id, body) {
|
|
649
698
|
if (!id)
|
|
650
|
-
throw new Error(
|
|
699
|
+
throw new Error("id is required");
|
|
651
700
|
const payload = {
|
|
652
701
|
run_id: body.run_id,
|
|
653
702
|
action: body.action,
|