@kweaver-ai/kweaver-sdk 0.7.2 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -1
- package/README.zh.md +26 -0
- package/bin/kweaver.js +12 -11
- package/dist/api/bkn-backend.d.ts +1 -0
- package/dist/api/bkn-backend.js +1 -1
- package/dist/api/bkn-metrics.d.ts +59 -0
- package/dist/api/bkn-metrics.js +129 -0
- package/dist/api/conversations.d.ts +47 -2
- package/dist/api/conversations.js +113 -17
- package/dist/api/datasources.d.ts +7 -0
- package/dist/api/datasources.js +51 -6
- package/dist/api/model-invocation.d.ts +58 -0
- package/dist/api/model-invocation.js +203 -0
- package/dist/api/models.d.ts +79 -0
- package/dist/api/models.js +183 -0
- package/dist/api/ontology-query-metrics.d.ts +14 -0
- package/dist/api/ontology-query-metrics.js +30 -0
- package/dist/api/toolboxes.d.ts +2 -0
- package/dist/api/toolboxes.js +2 -1
- package/dist/bundled-model-templates.d.ts +17 -0
- package/dist/bundled-model-templates.js +24 -0
- package/dist/cli.js +28 -2
- package/dist/client.d.ts +3 -0
- package/dist/client.js +5 -0
- package/dist/commands/agent.d.ts +7 -1
- package/dist/commands/agent.js +75 -21
- package/dist/commands/auth.js +42 -7
- package/dist/commands/bkn-metric.d.ts +1 -0
- package/dist/commands/bkn-metric.js +406 -0
- package/dist/commands/bkn-ops.d.ts +2 -1
- package/dist/commands/bkn-ops.js +75 -34
- package/dist/commands/bkn-utils.d.ts +55 -2
- package/dist/commands/bkn-utils.js +103 -9
- package/dist/commands/bkn.js +4 -0
- package/dist/commands/dataflow.js +194 -20
- package/dist/commands/ds.d.ts +0 -1
- package/dist/commands/ds.js +26 -10
- package/dist/commands/explore-chat.js +2 -2
- package/dist/commands/import-csv.d.ts +0 -2
- package/dist/commands/import-csv.js +2 -4
- package/dist/commands/model.d.ts +72 -0
- package/dist/commands/model.js +1315 -0
- package/dist/commands/tool.d.ts +1 -0
- package/dist/commands/tool.js +12 -0
- package/dist/config/store.d.ts +1 -0
- package/dist/config/store.js +17 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +5 -0
- package/dist/resources/models.d.ts +40 -0
- package/dist/resources/models.js +88 -0
- package/dist/resources/toolboxes.d.ts +2 -0
- package/dist/templates/bkn/document/manifest.json +12 -0
- package/dist/templates/bkn/document/template.json +757 -0
- package/dist/templates/dataflow/unstructured/manifest.json +11 -0
- package/dist/templates/dataflow/unstructured/template.json +63 -0
- package/dist/templates/dataset/document/manifest.json +10 -0
- package/dist/templates/dataset/document/template.json +23 -0
- package/dist/templates/dataset/document-content/manifest.json +10 -0
- package/dist/templates/dataset/document-content/template.json +29 -0
- package/dist/templates/dataset/document-element/manifest.json +10 -0
- package/dist/templates/dataset/document-element/template.json +21 -0
- package/dist/templates/model/llm-basic.json +13 -0
- package/dist/templates/model/manifest.json +16 -0
- package/dist/templates/model/small-basic.json +6 -0
- package/dist/utils/template-loader.d.ts +40 -0
- package/dist/utils/template-loader.js +129 -0
- package/dist/utils/trace-views.d.ts +44 -0
- package/dist/utils/trace-views.js +425 -0
- package/package.json +3 -3
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
import { ensureValidToken, formatHttpError } from "../auth/oauth.js";
|
|
2
|
+
import { listMetrics, createMetrics, searchMetrics, validateMetrics, getMetric, updateMetric, deleteMetric, getMetrics, deleteMetrics, } from "../api/bkn-metrics.js";
|
|
3
|
+
import { metricQueryData, metricDryRun } from "../api/ontology-query-metrics.js";
|
|
4
|
+
import { formatCallOutput } from "./call.js";
|
|
5
|
+
import { resolveBusinessDomain } from "../config/store.js";
|
|
6
|
+
import { parseJsonObject, parseSearchAfterArray, confirmYes } from "./bkn-utils.js";
|
|
7
|
+
function parseCommaSeparatedIds(raw) {
|
|
8
|
+
return raw
|
|
9
|
+
.split(",")
|
|
10
|
+
.map((s) => s.trim())
|
|
11
|
+
.filter((s) => s.length > 0);
|
|
12
|
+
}
|
|
13
|
+
const METRIC_HELP = `kweaver bkn metric <action> [args] [--pretty] [-bd <domain>]
|
|
14
|
+
|
|
15
|
+
Management (bkn-backend):
|
|
16
|
+
list <kn-id> [--limit <n>] [--branch <b>] [--name-pattern <p>] [--sort update_time|name] [--direction asc|desc] [--offset <n>] [--tag <t>] [--group-id <id>]
|
|
17
|
+
get <kn-id> <metric-id(s)> [--branch <b>] (comma-separated for multiple)
|
|
18
|
+
create <kn-id> '<json>' [--branch] [--strict-mode true|false]
|
|
19
|
+
search <kn-id> '<json>' [--branch] [--strict-mode] [--limit <n>] [--search-after '<json>']
|
|
20
|
+
validate <kn-id> '<json>' [--branch] [--strict-mode] [--import-mode normal|ignore|overwrite]
|
|
21
|
+
update <kn-id> <metric-id> '<json>' [--branch] [--strict-mode]
|
|
22
|
+
delete <kn-id> <metric-id(s)> [-y] (comma-separated for multiple)
|
|
23
|
+
|
|
24
|
+
Query (ontology-query):
|
|
25
|
+
query <kn-id> <metric-id> ['<json-body>'] [--branch] [--fill-null]
|
|
26
|
+
dry-run <kn-id> '<json>' [--branch] [--fill-null]
|
|
27
|
+
|
|
28
|
+
list: default --limit 30. search/query JSON: default limit 50 in body when not set.`;
|
|
29
|
+
function parseListArgs(args) {
|
|
30
|
+
let pretty = true;
|
|
31
|
+
let businessDomain = "";
|
|
32
|
+
let limit = 30;
|
|
33
|
+
let branch;
|
|
34
|
+
let namePattern;
|
|
35
|
+
let sort;
|
|
36
|
+
let direction;
|
|
37
|
+
let offset;
|
|
38
|
+
let tag;
|
|
39
|
+
let groupId;
|
|
40
|
+
const pos = [];
|
|
41
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
42
|
+
const a = args[i];
|
|
43
|
+
if (a === "--help" || a === "-h")
|
|
44
|
+
throw new Error("help");
|
|
45
|
+
if (a === "--pretty") {
|
|
46
|
+
pretty = true;
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
if ((a === "-bd" || a === "--biz-domain") && args[i + 1]) {
|
|
50
|
+
businessDomain = args[i + 1];
|
|
51
|
+
i += 1;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (a === "--limit" && args[i + 1]) {
|
|
55
|
+
const n = parseInt(args[i + 1], 10);
|
|
56
|
+
if (Number.isNaN(n) || n < 1)
|
|
57
|
+
throw new Error("Invalid --limit");
|
|
58
|
+
limit = n;
|
|
59
|
+
i += 1;
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
if (a === "--branch" && args[i + 1]) {
|
|
63
|
+
branch = args[i + 1];
|
|
64
|
+
i += 1;
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
if (a === "--name-pattern" && args[i + 1]) {
|
|
68
|
+
namePattern = args[i + 1];
|
|
69
|
+
i += 1;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (a === "--sort" && args[i + 1]) {
|
|
73
|
+
const s = args[i + 1];
|
|
74
|
+
if (s !== "update_time" && s !== "name")
|
|
75
|
+
throw new Error("--sort must be update_time|name");
|
|
76
|
+
sort = s;
|
|
77
|
+
i += 1;
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
if (a === "--direction" && args[i + 1]) {
|
|
81
|
+
const d = args[i + 1];
|
|
82
|
+
if (d !== "asc" && d !== "desc")
|
|
83
|
+
throw new Error("--direction must be asc|desc");
|
|
84
|
+
direction = d;
|
|
85
|
+
i += 1;
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (a === "--offset" && args[i + 1]) {
|
|
89
|
+
offset = parseInt(args[i + 1], 10);
|
|
90
|
+
i += 1;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (a === "--tag" && args[i + 1]) {
|
|
94
|
+
tag = args[i + 1];
|
|
95
|
+
i += 1;
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (a === "--group-id" && args[i + 1]) {
|
|
99
|
+
groupId = args[i + 1];
|
|
100
|
+
i += 1;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
pos.push(a);
|
|
104
|
+
}
|
|
105
|
+
if (!businessDomain)
|
|
106
|
+
businessDomain = resolveBusinessDomain();
|
|
107
|
+
const [knId] = pos;
|
|
108
|
+
if (!knId)
|
|
109
|
+
throw new Error("Usage: kweaver bkn metric list <kn-id> [options]");
|
|
110
|
+
return {
|
|
111
|
+
knId,
|
|
112
|
+
limit,
|
|
113
|
+
pretty,
|
|
114
|
+
businessDomain,
|
|
115
|
+
branch,
|
|
116
|
+
namePattern,
|
|
117
|
+
sort,
|
|
118
|
+
direction,
|
|
119
|
+
offset,
|
|
120
|
+
tag,
|
|
121
|
+
groupId,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
function parseCommonKnFlags(rest, withYes) {
|
|
125
|
+
let pretty = true;
|
|
126
|
+
let businessDomain = "";
|
|
127
|
+
let branch;
|
|
128
|
+
let strictMode;
|
|
129
|
+
let importMode;
|
|
130
|
+
let fillNull;
|
|
131
|
+
let yes = false;
|
|
132
|
+
const out = [];
|
|
133
|
+
for (let i = 0; i < rest.length; i += 1) {
|
|
134
|
+
const a = rest[i];
|
|
135
|
+
if (a === "--help" || a === "-h")
|
|
136
|
+
throw new Error("help");
|
|
137
|
+
if (a === "--pretty") {
|
|
138
|
+
pretty = true;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (withYes && (a === "-y" || a === "--yes")) {
|
|
142
|
+
yes = true;
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if ((a === "-bd" || a === "--biz-domain") && rest[i + 1]) {
|
|
146
|
+
businessDomain = rest[i + 1];
|
|
147
|
+
i += 1;
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
if (a === "--branch" && rest[i + 1]) {
|
|
151
|
+
branch = rest[i + 1];
|
|
152
|
+
i += 1;
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
if (a === "--strict-mode" && rest[i + 1]) {
|
|
156
|
+
strictMode = rest[i + 1] === "true" || rest[i + 1] === "1";
|
|
157
|
+
i += 1;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
if (a === "--import-mode" && rest[i + 1]) {
|
|
161
|
+
const m = rest[i + 1];
|
|
162
|
+
if (m !== "normal" && m !== "ignore" && m !== "overwrite") {
|
|
163
|
+
throw new Error("--import-mode must be normal|ignore|overwrite");
|
|
164
|
+
}
|
|
165
|
+
importMode = m;
|
|
166
|
+
i += 1;
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
if (a === "--fill-null") {
|
|
170
|
+
fillNull = true;
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
out.push(a);
|
|
174
|
+
}
|
|
175
|
+
if (!businessDomain)
|
|
176
|
+
businessDomain = resolveBusinessDomain();
|
|
177
|
+
return { filtered: out, pretty, businessDomain, branch, strictMode, importMode, fillNull, yes };
|
|
178
|
+
}
|
|
179
|
+
export async function runKnMetricCommand(args) {
|
|
180
|
+
const [action, ...rest] = args;
|
|
181
|
+
if (!action || action === "--help" || action === "-h") {
|
|
182
|
+
console.log(METRIC_HELP);
|
|
183
|
+
return 0;
|
|
184
|
+
}
|
|
185
|
+
try {
|
|
186
|
+
const token = await ensureValidToken();
|
|
187
|
+
const b = { baseUrl: token.baseUrl, accessToken: token.accessToken };
|
|
188
|
+
if (action === "list") {
|
|
189
|
+
const p = parseListArgs(rest);
|
|
190
|
+
const out = await listMetrics({
|
|
191
|
+
...b,
|
|
192
|
+
knId: p.knId,
|
|
193
|
+
businessDomain: p.businessDomain,
|
|
194
|
+
limit: p.limit,
|
|
195
|
+
branch: p.branch,
|
|
196
|
+
namePattern: p.namePattern,
|
|
197
|
+
sort: p.sort,
|
|
198
|
+
direction: p.direction,
|
|
199
|
+
offset: p.offset,
|
|
200
|
+
tag: p.tag,
|
|
201
|
+
groupId: p.groupId,
|
|
202
|
+
});
|
|
203
|
+
console.log(formatCallOutput(out, p.pretty));
|
|
204
|
+
return 0;
|
|
205
|
+
}
|
|
206
|
+
if (action === "get") {
|
|
207
|
+
const o = parseCommonKnFlags(rest, false);
|
|
208
|
+
const [knId, metricIdArg] = o.filtered;
|
|
209
|
+
if (!knId || !metricIdArg) {
|
|
210
|
+
console.error("Usage: kweaver bkn metric get <kn-id> <metric-id(s)> [options]");
|
|
211
|
+
return 1;
|
|
212
|
+
}
|
|
213
|
+
const ids = parseCommaSeparatedIds(metricIdArg);
|
|
214
|
+
if (ids.length === 0) {
|
|
215
|
+
console.error("metric-id(s): need at least one id");
|
|
216
|
+
return 1;
|
|
217
|
+
}
|
|
218
|
+
const out = ids.length === 1
|
|
219
|
+
? await getMetric({ ...b, knId, businessDomain: o.businessDomain, metricId: ids[0], branch: o.branch })
|
|
220
|
+
: await getMetrics({ ...b, knId, businessDomain: o.businessDomain, metricIds: ids.join(","), branch: o.branch });
|
|
221
|
+
console.log(formatCallOutput(out, o.pretty));
|
|
222
|
+
return 0;
|
|
223
|
+
}
|
|
224
|
+
if (action === "create") {
|
|
225
|
+
const o = parseCommonKnFlags(rest, false);
|
|
226
|
+
const [knId, bodyJson] = o.filtered;
|
|
227
|
+
if (!knId || !bodyJson) {
|
|
228
|
+
console.error("Usage: kweaver bkn metric create <kn-id> '<json>' [options]");
|
|
229
|
+
return 1;
|
|
230
|
+
}
|
|
231
|
+
const out = await createMetrics({
|
|
232
|
+
...b,
|
|
233
|
+
knId,
|
|
234
|
+
businessDomain: o.businessDomain,
|
|
235
|
+
body: bodyJson,
|
|
236
|
+
branch: o.branch,
|
|
237
|
+
strictMode: o.strictMode,
|
|
238
|
+
});
|
|
239
|
+
console.log(formatCallOutput(out, o.pretty));
|
|
240
|
+
return 0;
|
|
241
|
+
}
|
|
242
|
+
if (action === "search") {
|
|
243
|
+
let limit;
|
|
244
|
+
let searchAfter;
|
|
245
|
+
const r = [];
|
|
246
|
+
for (let i = 0; i < rest.length; i += 1) {
|
|
247
|
+
const a = rest[i];
|
|
248
|
+
if (a === "--limit" && rest[i + 1]) {
|
|
249
|
+
limit = parseInt(rest[i + 1], 10);
|
|
250
|
+
if (Number.isNaN(limit) || limit < 1)
|
|
251
|
+
throw new Error("Invalid --limit");
|
|
252
|
+
i += 1;
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
if (a === "--search-after" && rest[i + 1]) {
|
|
256
|
+
searchAfter = parseSearchAfterArray(rest[i + 1]);
|
|
257
|
+
i += 1;
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
r.push(a);
|
|
261
|
+
}
|
|
262
|
+
const o = parseCommonKnFlags(r, false);
|
|
263
|
+
const [knId, bodyText] = o.filtered;
|
|
264
|
+
if (!knId || !bodyText) {
|
|
265
|
+
console.error("Usage: kweaver bkn metric search <kn-id> '<json>' [options]");
|
|
266
|
+
return 1;
|
|
267
|
+
}
|
|
268
|
+
const obj = parseJsonObject(bodyText, "search body must be a JSON object.");
|
|
269
|
+
if (limit !== undefined)
|
|
270
|
+
obj.limit = limit;
|
|
271
|
+
if (searchAfter !== undefined)
|
|
272
|
+
obj.search_after = searchAfter;
|
|
273
|
+
if (typeof obj.limit !== "number" || !Number.isFinite(obj.limit)) {
|
|
274
|
+
obj.limit = 50;
|
|
275
|
+
}
|
|
276
|
+
const out = await searchMetrics({
|
|
277
|
+
...b,
|
|
278
|
+
knId,
|
|
279
|
+
businessDomain: o.businessDomain,
|
|
280
|
+
body: JSON.stringify(obj),
|
|
281
|
+
branch: o.branch,
|
|
282
|
+
strictMode: o.strictMode,
|
|
283
|
+
});
|
|
284
|
+
console.log(formatCallOutput(out, o.pretty));
|
|
285
|
+
return 0;
|
|
286
|
+
}
|
|
287
|
+
if (action === "validate") {
|
|
288
|
+
const o = parseCommonKnFlags(rest, false);
|
|
289
|
+
const [knId, bodyJson] = o.filtered;
|
|
290
|
+
if (!knId || !bodyJson) {
|
|
291
|
+
console.error("Usage: kweaver bkn metric validate <kn-id> '<json>' [options]");
|
|
292
|
+
return 1;
|
|
293
|
+
}
|
|
294
|
+
const out = await validateMetrics({
|
|
295
|
+
...b,
|
|
296
|
+
knId,
|
|
297
|
+
businessDomain: o.businessDomain,
|
|
298
|
+
body: bodyJson,
|
|
299
|
+
branch: o.branch,
|
|
300
|
+
strictMode: o.strictMode,
|
|
301
|
+
importMode: o.importMode,
|
|
302
|
+
});
|
|
303
|
+
console.log(formatCallOutput(out, o.pretty));
|
|
304
|
+
return 0;
|
|
305
|
+
}
|
|
306
|
+
if (action === "update") {
|
|
307
|
+
const o = parseCommonKnFlags(rest, false);
|
|
308
|
+
const [knId, metricId, bodyJson] = o.filtered;
|
|
309
|
+
if (!knId || !metricId || !bodyJson) {
|
|
310
|
+
console.error("Usage: kweaver bkn metric update <kn-id> <metric-id> '<json>' [options]");
|
|
311
|
+
return 1;
|
|
312
|
+
}
|
|
313
|
+
const out = await updateMetric({
|
|
314
|
+
...b,
|
|
315
|
+
knId,
|
|
316
|
+
businessDomain: o.businessDomain,
|
|
317
|
+
metricId,
|
|
318
|
+
body: bodyJson,
|
|
319
|
+
branch: o.branch,
|
|
320
|
+
strictMode: o.strictMode,
|
|
321
|
+
});
|
|
322
|
+
console.log(formatCallOutput(out, o.pretty));
|
|
323
|
+
return 0;
|
|
324
|
+
}
|
|
325
|
+
if (action === "delete") {
|
|
326
|
+
const o = parseCommonKnFlags(rest, true);
|
|
327
|
+
const [knId, metricIdArg] = o.filtered;
|
|
328
|
+
if (!knId || !metricIdArg) {
|
|
329
|
+
console.error("Usage: kweaver bkn metric delete <kn-id> <metric-id(s)> [-y]");
|
|
330
|
+
return 1;
|
|
331
|
+
}
|
|
332
|
+
const ids = parseCommaSeparatedIds(metricIdArg);
|
|
333
|
+
if (ids.length === 0) {
|
|
334
|
+
console.error("metric-id(s): need at least one id");
|
|
335
|
+
return 1;
|
|
336
|
+
}
|
|
337
|
+
if (!o.yes) {
|
|
338
|
+
const label = ids.length === 1 ? ids[0] : ids.join(",");
|
|
339
|
+
const ok = await confirmYes(`Delete metric(s) ${label}?`);
|
|
340
|
+
if (!ok) {
|
|
341
|
+
console.log("Cancelled.");
|
|
342
|
+
return 0;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
const out = ids.length === 1
|
|
346
|
+
? await deleteMetric({ ...b, knId, businessDomain: o.businessDomain, metricId: ids[0], branch: o.branch })
|
|
347
|
+
: await deleteMetrics({ ...b, knId, businessDomain: o.businessDomain, metricIds: ids.join(","), branch: o.branch });
|
|
348
|
+
console.log(formatCallOutput(out, o.pretty));
|
|
349
|
+
return 0;
|
|
350
|
+
}
|
|
351
|
+
if (action === "query") {
|
|
352
|
+
const o = parseCommonKnFlags(rest, false);
|
|
353
|
+
const { fillNull, branch, filtered, pretty, businessDomain } = o;
|
|
354
|
+
const [knId, metricId, bodyText = "{}"] = filtered;
|
|
355
|
+
if (!knId || !metricId) {
|
|
356
|
+
console.error("Usage: kweaver bkn metric query <kn-id> <metric-id> ['<json>'] [--branch] [--fill-null]");
|
|
357
|
+
return 1;
|
|
358
|
+
}
|
|
359
|
+
const body = parseJsonObject(bodyText, "metric query body must be a JSON object.");
|
|
360
|
+
if (typeof body.limit !== "number" || !Number.isFinite(body.limit)) {
|
|
361
|
+
body.limit = 50;
|
|
362
|
+
}
|
|
363
|
+
const out = await metricQueryData({
|
|
364
|
+
...b,
|
|
365
|
+
knId,
|
|
366
|
+
businessDomain,
|
|
367
|
+
metricId,
|
|
368
|
+
body: JSON.stringify(body),
|
|
369
|
+
branch,
|
|
370
|
+
fillNull,
|
|
371
|
+
});
|
|
372
|
+
console.log(formatCallOutput(out, pretty));
|
|
373
|
+
return 0;
|
|
374
|
+
}
|
|
375
|
+
if (action === "dry-run") {
|
|
376
|
+
const o = parseCommonKnFlags(rest, false);
|
|
377
|
+
const { fillNull, branch, filtered, pretty, businessDomain } = o;
|
|
378
|
+
const [knId, bodyText] = filtered;
|
|
379
|
+
if (!knId || !bodyText) {
|
|
380
|
+
console.error("Usage: kweaver bkn metric dry-run <kn-id> '<json>' [--branch] [--fill-null]");
|
|
381
|
+
return 1;
|
|
382
|
+
}
|
|
383
|
+
parseJsonObject(bodyText, "dry-run body must be a JSON object.");
|
|
384
|
+
const out = await metricDryRun({
|
|
385
|
+
...b,
|
|
386
|
+
knId,
|
|
387
|
+
businessDomain,
|
|
388
|
+
body: bodyText,
|
|
389
|
+
branch,
|
|
390
|
+
fillNull,
|
|
391
|
+
});
|
|
392
|
+
console.log(formatCallOutput(out, pretty));
|
|
393
|
+
return 0;
|
|
394
|
+
}
|
|
395
|
+
console.error(`Unknown bkn metric action: ${action}. Use --help.`);
|
|
396
|
+
return 1;
|
|
397
|
+
}
|
|
398
|
+
catch (error) {
|
|
399
|
+
if (error instanceof Error && error.message === "help") {
|
|
400
|
+
console.log(METRIC_HELP);
|
|
401
|
+
return 0;
|
|
402
|
+
}
|
|
403
|
+
console.error(formatHttpError(error));
|
|
404
|
+
return 1;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
@@ -32,6 +32,7 @@ export declare function parseKnCreateFromDsArgs(args: string[]): {
|
|
|
32
32
|
dsId: string;
|
|
33
33
|
name: string;
|
|
34
34
|
tables: string[];
|
|
35
|
+
pkMap: Record<string, string>;
|
|
35
36
|
build: boolean;
|
|
36
37
|
timeout: number;
|
|
37
38
|
businessDomain: string;
|
|
@@ -51,8 +52,8 @@ export declare function parseKnCreateFromCsvArgs(args: string[]): {
|
|
|
51
52
|
tablePrefix: string;
|
|
52
53
|
batchSize: number;
|
|
53
54
|
tables: string[];
|
|
55
|
+
pkMap: Record<string, string>;
|
|
54
56
|
build: boolean;
|
|
55
|
-
recreate: boolean;
|
|
56
57
|
timeout: number;
|
|
57
58
|
businessDomain: string;
|
|
58
59
|
noRollback: boolean;
|
package/dist/commands/bkn-ops.js
CHANGED
|
@@ -5,7 +5,7 @@ import { loadNetwork, allObjects, allRelations, allActions, generateChecksum, va
|
|
|
5
5
|
import { prepareBknDirectoryForImport, stripBknEncodingCliArgs, } from "../utils/bkn-encoding.js";
|
|
6
6
|
import { ensureValidToken, formatHttpError } from "../auth/oauth.js";
|
|
7
7
|
import { createKnowledgeNetwork, createObjectTypes, deleteKnowledgeNetwork, buildKnowledgeNetwork, getBuildStatus, } from "../api/knowledge-networks.js";
|
|
8
|
-
import { listTablesWithColumns,
|
|
8
|
+
import { listTablesWithColumns, scanDatasourceMetadata } from "../api/datasources.js";
|
|
9
9
|
import { createDataView, findDataView } from "../api/dataviews.js";
|
|
10
10
|
import { resolveFiles } from "./ds.js";
|
|
11
11
|
import { buildTableName } from "./import-csv.js";
|
|
@@ -13,7 +13,7 @@ import { downloadBkn, uploadBkn, listActionSchedules, getActionSchedule, createA
|
|
|
13
13
|
import { formatCallOutput } from "./call.js";
|
|
14
14
|
import { resolveBusinessDomain } from "../config/store.js";
|
|
15
15
|
import { runDsImportCsv } from "./ds.js";
|
|
16
|
-
import { pollWithBackoff,
|
|
16
|
+
import { pollWithBackoff, detectDisplayKey, formatPkDetectionError, parsePkMap, resolvePrimaryKey, confirmYes, } from "./bkn-utils.js";
|
|
17
17
|
// ── BKN object name validation ──────────────────────────────────────────────
|
|
18
18
|
// Mirrors bkn-backend OBJECT_NAME_MAX_LENGTH (interfaces/common.go:28) and
|
|
19
19
|
// validateObjectName (driveradapters/validate.go:85). 40 utf-8 codepoints,
|
|
@@ -480,6 +480,8 @@ Create a knowledge network from a datasource (dataviews + object types + optiona
|
|
|
480
480
|
Options:
|
|
481
481
|
--name <s> Knowledge network name (required)
|
|
482
482
|
--tables <a,b> Comma-separated table names (default: all)
|
|
483
|
+
--pk-map <s> Explicit primary keys: <table>:<field>[,<table>:<field>...]
|
|
484
|
+
Required when auto-detection fails (no unique column in sample)
|
|
483
485
|
--build (default) Build after creation
|
|
484
486
|
--no-build Skip build after creation
|
|
485
487
|
--timeout <n> Build timeout in seconds (default: 300)
|
|
@@ -490,6 +492,7 @@ export function parseKnCreateFromDsArgs(args) {
|
|
|
490
492
|
let dsId = "";
|
|
491
493
|
let name = "";
|
|
492
494
|
let tablesStr = "";
|
|
495
|
+
let pkMapStr = "";
|
|
493
496
|
let build = true;
|
|
494
497
|
let timeout = 300;
|
|
495
498
|
let businessDomain = "";
|
|
@@ -507,6 +510,10 @@ export function parseKnCreateFromDsArgs(args) {
|
|
|
507
510
|
tablesStr = args[++i];
|
|
508
511
|
continue;
|
|
509
512
|
}
|
|
513
|
+
if (arg === "--pk-map" && args[i + 1]) {
|
|
514
|
+
pkMapStr = args[++i];
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
510
517
|
if (arg === "--build") {
|
|
511
518
|
build = true;
|
|
512
519
|
continue;
|
|
@@ -541,9 +548,10 @@ export function parseKnCreateFromDsArgs(args) {
|
|
|
541
548
|
if (!dsId || !name) {
|
|
542
549
|
throw new Error("Usage: kweaver bkn create-from-ds <ds-id> --name X [options]");
|
|
543
550
|
}
|
|
551
|
+
const pkMap = pkMapStr ? parsePkMap(pkMapStr) : {};
|
|
544
552
|
if (!businessDomain)
|
|
545
553
|
businessDomain = resolveBusinessDomain();
|
|
546
|
-
return { dsId, name, tables, build, timeout, businessDomain, pretty, noRollback };
|
|
554
|
+
return { dsId, name, tables, pkMap, build, timeout, businessDomain, pretty, noRollback };
|
|
547
555
|
}
|
|
548
556
|
/** Sanitize a table name into a BKN-safe ID (alphanumeric + underscore). */
|
|
549
557
|
function sanitizeBknId(name) {
|
|
@@ -587,6 +595,7 @@ export async function runKnCreateFromDsCommand(args, sampleRows) {
|
|
|
587
595
|
const tableRetryDelayMs = 4000;
|
|
588
596
|
let allTables = [];
|
|
589
597
|
let targetTables = [];
|
|
598
|
+
let scanAttempted = false;
|
|
590
599
|
for (let attempt = 1; attempt <= maxTableListAttempts; attempt += 1) {
|
|
591
600
|
const tablesBody = await listTablesWithColumns({ ...base, id: options.dsId });
|
|
592
601
|
allTables = JSON.parse(tablesBody);
|
|
@@ -596,8 +605,24 @@ export async function runKnCreateFromDsCommand(args, sampleRows) {
|
|
|
596
605
|
if (targetTables.length > 0)
|
|
597
606
|
break;
|
|
598
607
|
if (attempt < maxTableListAttempts) {
|
|
599
|
-
|
|
600
|
-
|
|
608
|
+
// First miss: the catalog often hasn't picked up tables created
|
|
609
|
+
// out-of-band (e.g. ds import-csv from an older SDK that didn't
|
|
610
|
+
// self-scan). Trigger a scan once before falling back to plain
|
|
611
|
+
// sleep-retries.
|
|
612
|
+
if (!scanAttempted) {
|
|
613
|
+
scanAttempted = true;
|
|
614
|
+
console.error(`No tables available (attempt ${attempt}/${maxTableListAttempts}); scanning datasource metadata before retry...`);
|
|
615
|
+
try {
|
|
616
|
+
await scanDatasourceMetadata({ ...base, id: options.dsId });
|
|
617
|
+
}
|
|
618
|
+
catch (err) {
|
|
619
|
+
console.error(`Scan warning (continuing): ${formatHttpError(err)}`);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
else {
|
|
623
|
+
console.error(`No tables available (attempt ${attempt}/${maxTableListAttempts}); retrying in ${tableRetryDelayMs / 1000}s...`);
|
|
624
|
+
await new Promise((r) => setTimeout(r, tableRetryDelayMs));
|
|
625
|
+
}
|
|
601
626
|
}
|
|
602
627
|
}
|
|
603
628
|
if (targetTables.length === 0) {
|
|
@@ -608,6 +633,37 @@ export async function runKnCreateFromDsCommand(args, sampleRows) {
|
|
|
608
633
|
// Backend rejects the whole batch on first violation (validate.go:90),
|
|
609
634
|
// so retroactive rollback is wasted work if we can fail fast here.
|
|
610
635
|
assertValidBknObjectNames(targetTables.map((t) => t.name), "Object type names derived from table names");
|
|
636
|
+
// Pre-flight: resolve PK for every table BEFORE any side effect.
|
|
637
|
+
// Auto-detection silently picking the wrong column was the cause of
|
|
638
|
+
// issue #97 (KN built with ~5 indexed docs out of 2036 source rows).
|
|
639
|
+
// Resolve order: --pk-map override → cardinality-based detection → fail-fast.
|
|
640
|
+
const tablePks = {};
|
|
641
|
+
const unknownPkMapTables = Object.keys(options.pkMap).filter((name) => !targetTables.some((t) => t.name === name));
|
|
642
|
+
if (unknownPkMapTables.length > 0) {
|
|
643
|
+
throw new Error(`--pk-map references unknown table(s): ${unknownPkMapTables.join(", ")}`);
|
|
644
|
+
}
|
|
645
|
+
for (const t of targetTables) {
|
|
646
|
+
const override = options.pkMap[t.name];
|
|
647
|
+
if (override && !t.columns.some((c) => c.name === override)) {
|
|
648
|
+
throw new Error(`--pk-map specifies '${override}' for table '${t.name}', but no such column. ` +
|
|
649
|
+
`Columns: ${t.columns.map((c) => c.name).join(", ")}`);
|
|
650
|
+
}
|
|
651
|
+
const resolution = resolvePrimaryKey(t, sampleRows?.[t.name], override);
|
|
652
|
+
if (resolution.pk) {
|
|
653
|
+
tablePks[t.name] = resolution.pk;
|
|
654
|
+
continue;
|
|
655
|
+
}
|
|
656
|
+
if (resolution.source === "ambiguous") {
|
|
657
|
+
const cols = (resolution.ambiguous ?? []).join(", ");
|
|
658
|
+
throw new Error(`Table '${t.name}' has a composite PRIMARY KEY (${cols}). ` +
|
|
659
|
+
`BKN object types take a single primary key — pick one with --pk-map ${t.name}:<column>.`);
|
|
660
|
+
}
|
|
661
|
+
throw new Error(formatPkDetectionError(t.name, {
|
|
662
|
+
pk: null,
|
|
663
|
+
candidates: resolution.candidates ?? [],
|
|
664
|
+
sampleSize: resolution.sampleSize ?? 0,
|
|
665
|
+
}));
|
|
666
|
+
}
|
|
611
667
|
// Phase 1: Create DataViews for each table. findDataView is idempotent;
|
|
612
668
|
// not tracked for rollback so a retry can reuse what's already there.
|
|
613
669
|
console.error(`Creating data views for ${targetTables.length} table(s) ...`);
|
|
@@ -653,7 +709,7 @@ export async function runKnCreateFromDsCommand(args, sampleRows) {
|
|
|
653
709
|
// (object_type_service.go:213-355) — all-or-nothing.
|
|
654
710
|
console.error(`Creating ${targetTables.length} object type(s) ...`);
|
|
655
711
|
const entries = targetTables.map((t) => {
|
|
656
|
-
const pk =
|
|
712
|
+
const pk = tablePks[t.name];
|
|
657
713
|
const dk = detectDisplayKey(t, pk);
|
|
658
714
|
return {
|
|
659
715
|
branch: "main",
|
|
@@ -759,7 +815,7 @@ Options:
|
|
|
759
815
|
--tables <a,b> Tables to include in KN (default: all imported)
|
|
760
816
|
--build (default) Build after creation
|
|
761
817
|
--no-build Skip build
|
|
762
|
-
--
|
|
818
|
+
--pk-map <s> Explicit primary keys: <table>:<field>[,<table>:<field>...]
|
|
763
819
|
--timeout <n> Build timeout in seconds (default: 300)
|
|
764
820
|
--no-rollback Keep partially-created KN on failure (debug; default: rollback)
|
|
765
821
|
-bd, --biz-domain Business domain (default: bd_public)`;
|
|
@@ -770,8 +826,8 @@ export function parseKnCreateFromCsvArgs(args) {
|
|
|
770
826
|
let tablePrefix = "";
|
|
771
827
|
let batchSize = 500;
|
|
772
828
|
let tablesStr = "";
|
|
829
|
+
let pkMapStr = "";
|
|
773
830
|
let build = true;
|
|
774
|
-
let recreate = false;
|
|
775
831
|
let timeout = 300;
|
|
776
832
|
let businessDomain = "";
|
|
777
833
|
let noRollback = false;
|
|
@@ -809,8 +865,8 @@ export function parseKnCreateFromCsvArgs(args) {
|
|
|
809
865
|
build = false;
|
|
810
866
|
continue;
|
|
811
867
|
}
|
|
812
|
-
if (arg === "--
|
|
813
|
-
|
|
868
|
+
if (arg === "--pk-map" && args[i + 1]) {
|
|
869
|
+
pkMapStr = args[++i];
|
|
814
870
|
continue;
|
|
815
871
|
}
|
|
816
872
|
if (arg === "--no-rollback") {
|
|
@@ -835,9 +891,10 @@ export function parseKnCreateFromCsvArgs(args) {
|
|
|
835
891
|
if (!dsId || !files || !name) {
|
|
836
892
|
throw new Error("Usage: kweaver bkn create-from-csv <ds-id> --files <glob> --name X [options]");
|
|
837
893
|
}
|
|
894
|
+
const pkMap = pkMapStr ? parsePkMap(pkMapStr) : {};
|
|
838
895
|
if (!businessDomain)
|
|
839
896
|
businessDomain = resolveBusinessDomain();
|
|
840
|
-
return { dsId, files, name, tablePrefix, batchSize, tables,
|
|
897
|
+
return { dsId, files, name, tablePrefix, batchSize, tables, pkMap, build, timeout, businessDomain, noRollback };
|
|
841
898
|
}
|
|
842
899
|
export async function runKnCreateFromCsvCommand(args) {
|
|
843
900
|
let options;
|
|
@@ -874,35 +931,15 @@ export async function runKnCreateFromCsvCommand(args) {
|
|
|
874
931
|
"--table-prefix", options.tablePrefix,
|
|
875
932
|
"--batch-size", String(options.batchSize),
|
|
876
933
|
"-bd", options.businessDomain,
|
|
877
|
-
...(options.recreate ? ["--recreate"] : []),
|
|
878
934
|
];
|
|
879
935
|
const importResult = await runDsImportCsv(importArgs);
|
|
880
936
|
if (importResult.code !== 0) {
|
|
881
937
|
console.error("CSV import failed — aborting KN creation");
|
|
882
938
|
return importResult.code;
|
|
883
939
|
}
|
|
884
|
-
// Phase 1.5
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
const token = await ensureValidToken();
|
|
888
|
-
const dsBody = await getDatasource({
|
|
889
|
-
baseUrl: token.baseUrl,
|
|
890
|
-
accessToken: token.accessToken,
|
|
891
|
-
id: options.dsId,
|
|
892
|
-
businessDomain: options.businessDomain,
|
|
893
|
-
});
|
|
894
|
-
const dsParsed = JSON.parse(dsBody);
|
|
895
|
-
await scanMetadata({
|
|
896
|
-
baseUrl: token.baseUrl,
|
|
897
|
-
accessToken: token.accessToken,
|
|
898
|
-
id: options.dsId,
|
|
899
|
-
dsType: dsParsed.type ?? "mysql",
|
|
900
|
-
businessDomain: options.businessDomain,
|
|
901
|
-
});
|
|
902
|
-
}
|
|
903
|
-
catch (err) {
|
|
904
|
-
console.error(`Scan warning (continuing): ${String(err)}`);
|
|
905
|
-
}
|
|
940
|
+
// (Phase 1.5 metadata scan removed — runDsImportCsv now self-scans on
|
|
941
|
+
// success, and runKnCreateFromDsCommand's table-discovery retry triggers
|
|
942
|
+
// a scan if the catalog still lags. Two layers of fallback are enough.)
|
|
906
943
|
// Phase 2: Create KN from datasource
|
|
907
944
|
console.error("Phase 2: Creating knowledge network ...");
|
|
908
945
|
const tableNames = options.tables.length > 0 ? options.tables : importResult.tables;
|
|
@@ -910,6 +947,7 @@ export async function runKnCreateFromCsvCommand(args) {
|
|
|
910
947
|
console.error("No tables available for KN creation — aborting");
|
|
911
948
|
return 1;
|
|
912
949
|
}
|
|
950
|
+
const pkMapEntries = Object.entries(options.pkMap);
|
|
913
951
|
const knArgs = [
|
|
914
952
|
options.dsId,
|
|
915
953
|
"--name", options.name,
|
|
@@ -917,6 +955,9 @@ export async function runKnCreateFromCsvCommand(args) {
|
|
|
917
955
|
options.build ? "--build" : "--no-build",
|
|
918
956
|
"--timeout", String(options.timeout),
|
|
919
957
|
"-bd", options.businessDomain,
|
|
958
|
+
...(pkMapEntries.length > 0
|
|
959
|
+
? ["--pk-map", pkMapEntries.map(([t, f]) => `${t}:${f}`).join(",")]
|
|
960
|
+
: []),
|
|
920
961
|
...(options.noRollback ? ["--no-rollback"] : []),
|
|
921
962
|
];
|
|
922
963
|
return runKnCreateFromDsCommand(knArgs, importResult.sampleRows);
|