@tiangong-ai/cli 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -2
- package/README.md +59 -2
- package/dist/args.d.ts +13 -0
- package/dist/args.js +72 -0
- package/dist/args.js.map +1 -0
- package/dist/cli.d.ts +5 -21
- package/dist/cli.js +46 -440
- package/dist/cli.js.map +1 -1
- package/dist/data.d.ts +3 -0
- package/dist/data.js +16 -0
- package/dist/data.js.map +1 -0
- package/dist/edge-search.d.ts +54 -0
- package/dist/edge-search.js +108 -0
- package/dist/edge-search.js.map +1 -0
- package/dist/education/commands.d.ts +2 -0
- package/dist/education/commands.js +144 -0
- package/dist/education/commands.js.map +1 -0
- package/dist/education/config.d.ts +24 -0
- package/dist/education/config.js +127 -0
- package/dist/education/config.js.map +1 -0
- package/dist/education/types.d.ts +1 -0
- package/dist/education/types.js +2 -0
- package/dist/education/types.js.map +1 -0
- package/dist/env.d.ts +2 -0
- package/dist/env.js +42 -0
- package/dist/env.js.map +1 -0
- package/dist/errors.d.ts +24 -0
- package/dist/errors.js +53 -0
- package/dist/errors.js.map +1 -0
- package/dist/http.d.ts +28 -0
- package/dist/http.js +80 -0
- package/dist/http.js.map +1 -0
- package/dist/io.d.ts +11 -0
- package/dist/io.js +33 -0
- package/dist/io.js.map +1 -0
- package/dist/kb/client.d.ts +19 -0
- package/dist/kb/client.js +71 -0
- package/dist/kb/client.js.map +1 -0
- package/dist/kb/config.d.ts +12 -0
- package/dist/kb/config.js +26 -0
- package/dist/kb/config.js.map +1 -0
- package/dist/kb/pipeline-health.d.ts +10 -0
- package/dist/kb/pipeline-health.js +68 -0
- package/dist/kb/pipeline-health.js.map +1 -0
- package/dist/kb/selector.d.ts +3 -0
- package/dist/kb/selector.js +40 -0
- package/dist/kb/selector.js.map +1 -0
- package/dist/kb/status.d.ts +19 -0
- package/dist/kb/status.js +74 -0
- package/dist/kb/status.js.map +1 -0
- package/dist/research/commands.d.ts +2 -0
- package/dist/research/commands.js +147 -0
- package/dist/research/commands.js.map +1 -0
- package/dist/research/config.d.ts +23 -0
- package/dist/research/config.js +120 -0
- package/dist/research/config.js.map +1 -0
- package/dist/research/types.d.ts +1 -0
- package/dist/research/types.js +2 -0
- package/dist/research/types.js.map +1 -0
- package/dist/strict-args.d.ts +8 -0
- package/dist/strict-args.js +63 -0
- package/dist/strict-args.js.map +1 -0
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,14 +1,27 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { createReadStream, openAsBlob,
|
|
2
|
+
import { createReadStream, openAsBlob, statSync } from "node:fs";
|
|
3
3
|
import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
4
4
|
import { homedir, platform } from "node:os";
|
|
5
5
|
import { basename, dirname, extname, isAbsolute, join, posix as pathPosix, relative, resolve, sep, } from "node:path";
|
|
6
6
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
7
|
-
import { fileURLToPath } from "node:url";
|
|
8
7
|
import { deflateRawSync, inflateRawSync } from "node:zlib";
|
|
9
8
|
import sharp from "sharp";
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
import { getBoolean, getNonNegativeInteger, getPositiveInteger, getString, nonNegativeIntegerValue, parseArgs, positiveIntegerValue, positiveNumberValue, } from "./args.js";
|
|
10
|
+
import { isObject, responseData, stringField } from "./data.js";
|
|
11
|
+
import { firstEnv, loadDotenv } from "./env.js";
|
|
12
|
+
import { CliError, HttpError } from "./errors.js";
|
|
13
|
+
import { jsonRequest } from "./http.js";
|
|
14
|
+
import { runEducationCommand } from "./education/commands.js";
|
|
15
|
+
import { collectionKey, collectionPath, listCollections, resolveCollection, resolveSelectorFields, } from "./kb/client.js";
|
|
16
|
+
import { resolveKbConfig as resolveConfig } from "./kb/config.js";
|
|
17
|
+
import { readBulkPipelineHealth } from "./kb/pipeline-health.js";
|
|
18
|
+
import { resolveCollectionSelector } from "./kb/selector.js";
|
|
19
|
+
import { batchDocumentStatuses, getDocumentStatus } from "./kb/status.js";
|
|
20
|
+
import { write } from "./io.js";
|
|
21
|
+
import { runResearchCommand } from "./research/commands.js";
|
|
22
|
+
export { parseArgs } from "./args.js";
|
|
23
|
+
export { DEFAULT_API_BASE_URL, DEFAULT_API_PATH_PREFIX } from "./kb/config.js";
|
|
24
|
+
export { resolveCollectionSelector } from "./kb/selector.js";
|
|
12
25
|
const DEFAULT_RETRIES = 3;
|
|
13
26
|
const DEFAULT_BULK_WINDOW_SIZE = 100;
|
|
14
27
|
const DEFAULT_BULK_TOP_UP_MAX = 50;
|
|
@@ -20,7 +33,6 @@ const DEFAULT_BULK_DERIVED_DIR = ".tiangong-kb-ingest-derived";
|
|
|
20
33
|
const DOCX_TARGET_IMAGE_DPI = 300;
|
|
21
34
|
const DOCX_NORMALIZE_MIN_BYTES = 10 * 1024 * 1024;
|
|
22
35
|
const EMUS_PER_INCH = 914400;
|
|
23
|
-
const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
|
|
24
36
|
const BULK_FAILED_STATUSES = new Set(["failed", "deleted", "dead", "timeout"]);
|
|
25
37
|
const BULK_IN_FLIGHT_STATUSES = new Set(["uploaded", "waiting_for_index_flags", "uploading"]);
|
|
26
38
|
const BULK_SUPPORTED_EXTENSIONS = new Set([
|
|
@@ -39,24 +51,6 @@ const BULK_SUPPORTED_EXTENSIONS = new Set([
|
|
|
39
51
|
".xls",
|
|
40
52
|
".xlsx",
|
|
41
53
|
]);
|
|
42
|
-
class CliError extends Error {
|
|
43
|
-
exitCode;
|
|
44
|
-
constructor(message, exitCode = 1) {
|
|
45
|
-
super(message);
|
|
46
|
-
this.exitCode = exitCode;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
class HttpError extends CliError {
|
|
50
|
-
status;
|
|
51
|
-
retryAfterSeconds;
|
|
52
|
-
retryable;
|
|
53
|
-
constructor(message, status, retryAfterSeconds, retryable) {
|
|
54
|
-
super(message);
|
|
55
|
-
this.status = status;
|
|
56
|
-
this.retryAfterSeconds = retryAfterSeconds;
|
|
57
|
-
this.retryable = retryable;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
54
|
export async function runCli(argv, io) {
|
|
61
55
|
try {
|
|
62
56
|
loadDotenv(io.env);
|
|
@@ -119,6 +113,12 @@ export async function runCli(argv, io) {
|
|
|
119
113
|
write(io.stdout, kbHelp());
|
|
120
114
|
return 1;
|
|
121
115
|
}
|
|
116
|
+
if (command === "research") {
|
|
117
|
+
return await runResearchCommand(subcommand ? [subcommand, ...rest] : rest, io);
|
|
118
|
+
}
|
|
119
|
+
if (command === "education") {
|
|
120
|
+
return await runEducationCommand(subcommand ? [subcommand, ...rest] : rest, io);
|
|
121
|
+
}
|
|
122
122
|
throw new CliError(`Unknown command: ${command}`);
|
|
123
123
|
}
|
|
124
124
|
catch (error) {
|
|
@@ -129,63 +129,6 @@ export async function runCli(argv, io) {
|
|
|
129
129
|
throw error;
|
|
130
130
|
}
|
|
131
131
|
}
|
|
132
|
-
export function parseArgs(argv) {
|
|
133
|
-
const positionals = [];
|
|
134
|
-
const flags = new Map();
|
|
135
|
-
for (let index = 0; index < argv.length; index += 1) {
|
|
136
|
-
const arg = argv[index];
|
|
137
|
-
if (!arg)
|
|
138
|
-
continue;
|
|
139
|
-
if (!arg.startsWith("--")) {
|
|
140
|
-
positionals.push(arg);
|
|
141
|
-
continue;
|
|
142
|
-
}
|
|
143
|
-
const withoutPrefix = arg.slice(2);
|
|
144
|
-
const equalsIndex = withoutPrefix.indexOf("=");
|
|
145
|
-
if (equalsIndex >= 0) {
|
|
146
|
-
flags.set(withoutPrefix.slice(0, equalsIndex), withoutPrefix.slice(equalsIndex + 1));
|
|
147
|
-
continue;
|
|
148
|
-
}
|
|
149
|
-
const next = argv[index + 1];
|
|
150
|
-
if (next && !next.startsWith("--")) {
|
|
151
|
-
flags.set(withoutPrefix, next);
|
|
152
|
-
index += 1;
|
|
153
|
-
}
|
|
154
|
-
else {
|
|
155
|
-
flags.set(withoutPrefix, true);
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
return { positionals, flags };
|
|
159
|
-
}
|
|
160
|
-
export function resolveCollectionSelector(args, env) {
|
|
161
|
-
const selectors = [];
|
|
162
|
-
addSelector(selectors, "primary_collection_id", getString(args, "collection-id"));
|
|
163
|
-
addSelector(selectors, "collection_path", getString(args, "collection-path"));
|
|
164
|
-
addSelector(selectors, "collection_key", getString(args, "collection-key"));
|
|
165
|
-
addSelector(selectors, "collection_name", getString(args, "collection-name"));
|
|
166
|
-
if (selectors.length > 1) {
|
|
167
|
-
throw new CliError("Provide exactly one collection selector.");
|
|
168
|
-
}
|
|
169
|
-
if (selectors.length === 1)
|
|
170
|
-
return selectors[0];
|
|
171
|
-
const envName = firstEnv(env, "TIANGONG_KB_DEFAULT_COLLECTION_NAME");
|
|
172
|
-
if (envName)
|
|
173
|
-
return { field: "collection_name", value: envName };
|
|
174
|
-
const legacyName = firstEnv(env, "TIANGONG_KB_DEFAULT_COLLECTION_ID");
|
|
175
|
-
if (legacyName) {
|
|
176
|
-
if (isUuid(legacyName)) {
|
|
177
|
-
throw new CliError("TIANGONG_KB_DEFAULT_COLLECTION_ID is treated as a collection name. Use --collection-id for UUID uploads.");
|
|
178
|
-
}
|
|
179
|
-
return { field: "collection_name", value: legacyName };
|
|
180
|
-
}
|
|
181
|
-
const envPath = firstEnv(env, "TIANGONG_KB_DEFAULT_COLLECTION_PATH");
|
|
182
|
-
if (envPath)
|
|
183
|
-
return { field: "collection_path", value: envPath };
|
|
184
|
-
const envKey = firstEnv(env, "TIANGONG_KB_DEFAULT_COLLECTION_KEY");
|
|
185
|
-
if (envKey)
|
|
186
|
-
return { field: "collection_key", value: envKey };
|
|
187
|
-
throw new CliError("Missing collection selector. Provide --collection-name, --collection-key, --collection-path, --collection-id, or set TIANGONG_KB_DEFAULT_COLLECTION_NAME.");
|
|
188
|
-
}
|
|
189
132
|
async function doctor(argv, io) {
|
|
190
133
|
const args = parseArgs(argv);
|
|
191
134
|
const config = resolveConfig(args, io.env, { requireKey: false });
|
|
@@ -366,6 +309,7 @@ async function kbIngestBulkRun(argv, io) {
|
|
|
366
309
|
metadataMap,
|
|
367
310
|
env: io.env,
|
|
368
311
|
stdout: io.stdout,
|
|
312
|
+
stderr: io.stderr,
|
|
369
313
|
preflightOptions,
|
|
370
314
|
});
|
|
371
315
|
writeJsonOrText(io.stdout, args, result, () => formatBulkRunSummary(result));
|
|
@@ -420,6 +364,7 @@ async function kbIngestResume(argv, io) {
|
|
|
420
364
|
metadataMap: job.metadataMap,
|
|
421
365
|
env: io.env,
|
|
422
366
|
stdout: io.stdout,
|
|
367
|
+
stderr: io.stderr,
|
|
423
368
|
preflightOptions,
|
|
424
369
|
});
|
|
425
370
|
writeJsonOrText(io.stdout, args, result, () => formatBulkRunSummary(result));
|
|
@@ -508,12 +453,24 @@ async function runBulkLoop(input) {
|
|
|
508
453
|
let lastPipelineHealth;
|
|
509
454
|
while (true) {
|
|
510
455
|
polls += 1;
|
|
511
|
-
|
|
456
|
+
let statusPollError;
|
|
457
|
+
try {
|
|
458
|
+
await pollBulkStatuses(input.statePath, input.config);
|
|
459
|
+
}
|
|
460
|
+
catch (error) {
|
|
461
|
+
if (!isRecoverableBulkStatusPollError(error)) {
|
|
462
|
+
throw error;
|
|
463
|
+
}
|
|
464
|
+
statusPollError = error;
|
|
465
|
+
write(input.stderr, `warning: bulk status polling failed and will be retried: ${error.message}\n`);
|
|
466
|
+
}
|
|
512
467
|
const summary = await bulkJobSummary(input.statePath);
|
|
513
468
|
const capacity = Math.max(0, windowSize - summary.inflight);
|
|
514
469
|
lastPipelineHealth = await readBulkPipelineHealth(input.config, pollInterval, input.selectorFields);
|
|
515
470
|
await saveBulkPipelineHealth(input.statePath, lastPipelineHealth);
|
|
516
|
-
const uploadLimit =
|
|
471
|
+
const uploadLimit = statusPollError
|
|
472
|
+
? 0
|
|
473
|
+
: bulkUploadLimitForHealth(Math.min(capacity, topUpMax), lastPipelineHealth);
|
|
517
474
|
const effectiveUploadConcurrency = lastPipelineHealth.recommendedAction === "slow_down" ? 1 : uploadConcurrency;
|
|
518
475
|
if (uploadLimit > 0) {
|
|
519
476
|
const pending = await claimPendingBulkFiles(input.statePath, uploadLimit);
|
|
@@ -574,6 +531,9 @@ function bulkUploadLimitForHealth(limit, health) {
|
|
|
574
531
|
return Math.max(1, Math.ceil(limit / 2));
|
|
575
532
|
return limit;
|
|
576
533
|
}
|
|
534
|
+
function isRecoverableBulkStatusPollError(error) {
|
|
535
|
+
return error instanceof HttpError && error.retryable;
|
|
536
|
+
}
|
|
577
537
|
function bulkPollIntervalForHealth(baseSeconds, health) {
|
|
578
538
|
if (!health)
|
|
579
539
|
return baseSeconds;
|
|
@@ -800,114 +760,6 @@ function judgeBulkStatus(item) {
|
|
|
800
760
|
}
|
|
801
761
|
return { status: "uploaded" };
|
|
802
762
|
}
|
|
803
|
-
async function batchDocumentStatuses(config, documentIds) {
|
|
804
|
-
try {
|
|
805
|
-
const payload = await jsonRequest(config, "documents/status:batch", {
|
|
806
|
-
method: "POST",
|
|
807
|
-
body: JSON.stringify({ documentIds }),
|
|
808
|
-
});
|
|
809
|
-
return batchStatusItems(payload);
|
|
810
|
-
}
|
|
811
|
-
catch (error) {
|
|
812
|
-
if (error instanceof HttpError && error.status && [404, 405, 501].includes(error.status)) {
|
|
813
|
-
return Promise.all(documentIds.map(async (documentId) => statusItemFromPayload(await getDocumentStatus(config, documentId), documentId)));
|
|
814
|
-
}
|
|
815
|
-
throw error;
|
|
816
|
-
}
|
|
817
|
-
}
|
|
818
|
-
async function getDocumentStatus(config, documentId) {
|
|
819
|
-
return jsonRequest(config, `documents/${encodeURIComponent(documentId)}/status`);
|
|
820
|
-
}
|
|
821
|
-
function batchStatusItems(payload) {
|
|
822
|
-
const data = responseData(payload);
|
|
823
|
-
const items = isObject(data) && Array.isArray(data.documents)
|
|
824
|
-
? data.documents
|
|
825
|
-
: isObject(data) && Array.isArray(data.results)
|
|
826
|
-
? data.results
|
|
827
|
-
: Array.isArray(data)
|
|
828
|
-
? data
|
|
829
|
-
: [];
|
|
830
|
-
return items.filter(isObject).map((item) => batchStatusItemFromPayload(item));
|
|
831
|
-
}
|
|
832
|
-
function statusItemFromPayload(payload, fallbackDocumentId = "") {
|
|
833
|
-
const data = responseData(payload);
|
|
834
|
-
const item = isObject(data) ? data : {};
|
|
835
|
-
return {
|
|
836
|
-
documentId: stringField(item, "documentId") ?? stringField(item, "document_id") ?? fallbackDocumentId,
|
|
837
|
-
status: stringField(item, "status"),
|
|
838
|
-
terminal: typeof item.terminal === "boolean" ? item.terminal : undefined,
|
|
839
|
-
opensearchIndexed: typeof item.opensearchIndexed === "boolean" ? item.opensearchIndexed : undefined,
|
|
840
|
-
pineconeIndexed: typeof item.pineconeIndexed === "boolean" ? item.pineconeIndexed : undefined,
|
|
841
|
-
indexRecordCount: typeof item.indexRecordCount === "number" ? item.indexRecordCount : undefined,
|
|
842
|
-
lastError: typeof item.lastError === "string" ? item.lastError : undefined,
|
|
843
|
-
lastErrorStage: typeof item.lastErrorStage === "string" ? item.lastErrorStage : undefined,
|
|
844
|
-
raw: payload,
|
|
845
|
-
};
|
|
846
|
-
}
|
|
847
|
-
async function readBulkPipelineHealth(config, fallbackPollAfterSeconds, selectorFields = {}) {
|
|
848
|
-
try {
|
|
849
|
-
const payload = await jsonRequest(config, pipelineHealthPath(selectorFields));
|
|
850
|
-
return pipelineHealthFromPayload(payload, fallbackPollAfterSeconds);
|
|
851
|
-
}
|
|
852
|
-
catch (error) {
|
|
853
|
-
if (error instanceof HttpError && error.status && [404, 405, 501].includes(error.status)) {
|
|
854
|
-
return {
|
|
855
|
-
healthy: true,
|
|
856
|
-
pressure: "unknown",
|
|
857
|
-
recommendedAction: "continue",
|
|
858
|
-
recommendedPollAfterSeconds: fallbackPollAfterSeconds,
|
|
859
|
-
message: "Pipeline health endpoint is unavailable; continuing without backpressure.",
|
|
860
|
-
};
|
|
861
|
-
}
|
|
862
|
-
if (error instanceof HttpError) {
|
|
863
|
-
return {
|
|
864
|
-
healthy: false,
|
|
865
|
-
pressure: "paused",
|
|
866
|
-
recommendedAction: "pause_top_up",
|
|
867
|
-
recommendedPollAfterSeconds: error.retryAfterSeconds ?? Math.max(fallbackPollAfterSeconds, 30),
|
|
868
|
-
message: error.message,
|
|
869
|
-
};
|
|
870
|
-
}
|
|
871
|
-
return {
|
|
872
|
-
healthy: false,
|
|
873
|
-
pressure: "paused",
|
|
874
|
-
recommendedAction: "pause_top_up",
|
|
875
|
-
recommendedPollAfterSeconds: Math.max(fallbackPollAfterSeconds, 30),
|
|
876
|
-
message: error instanceof Error ? error.message : String(error),
|
|
877
|
-
};
|
|
878
|
-
}
|
|
879
|
-
}
|
|
880
|
-
function pipelineHealthPath(selectorFields) {
|
|
881
|
-
const params = new URLSearchParams();
|
|
882
|
-
for (const key of ["primary_collection_id", "collection_path", "collection_key"]) {
|
|
883
|
-
const value = selectorFields[key];
|
|
884
|
-
if (value)
|
|
885
|
-
params.set(key, value);
|
|
886
|
-
}
|
|
887
|
-
const query = params.toString();
|
|
888
|
-
return query ? `pipeline/health?${query}` : "pipeline/health";
|
|
889
|
-
}
|
|
890
|
-
function pipelineHealthFromPayload(payload, fallbackPollAfterSeconds) {
|
|
891
|
-
const data = responseData(payload);
|
|
892
|
-
if (!isObject(data)) {
|
|
893
|
-
throw new CliError("Pipeline health response did not contain an object payload.");
|
|
894
|
-
}
|
|
895
|
-
const action = stringField(data, "recommendedAction");
|
|
896
|
-
if (action !== "continue" && action !== "slow_down" && action !== "pause_top_up") {
|
|
897
|
-
throw new CliError("Pipeline health response did not contain a valid recommendedAction.");
|
|
898
|
-
}
|
|
899
|
-
const pressure = stringField(data, "pressure");
|
|
900
|
-
const pollAfter = Number(data.recommendedPollAfterSeconds);
|
|
901
|
-
return {
|
|
902
|
-
healthy: typeof data.healthy === "boolean" ? data.healthy : action === "continue",
|
|
903
|
-
pressure: pressure === "ok" || pressure === "degraded" || pressure === "paused" ? pressure : "unknown",
|
|
904
|
-
recommendedAction: action,
|
|
905
|
-
recommendedPollAfterSeconds: Number.isFinite(pollAfter) && pollAfter > 0 ? pollAfter : fallbackPollAfterSeconds,
|
|
906
|
-
checkedAt: stringField(data, "checkedAt"),
|
|
907
|
-
message: stringField(data, "message") ??
|
|
908
|
-
(isObject(data.indexPreflight) ? stringField(data.indexPreflight, "message") : undefined),
|
|
909
|
-
};
|
|
910
|
-
}
|
|
911
763
|
async function saveBulkPipelineHealth(statePath, health) {
|
|
912
764
|
const db = await openSqlite(statePath);
|
|
913
765
|
const now = new Date().toISOString();
|
|
@@ -919,25 +771,6 @@ async function saveBulkPipelineHealth(statePath, health) {
|
|
|
919
771
|
db.close();
|
|
920
772
|
}
|
|
921
773
|
}
|
|
922
|
-
function batchStatusItemFromPayload(item) {
|
|
923
|
-
const documentId = stringField(item, "documentId") ?? stringField(item, "document_id") ?? "";
|
|
924
|
-
if (item.ok === true && isObject(item.status)) {
|
|
925
|
-
return statusItemFromPayload(item.status, documentId);
|
|
926
|
-
}
|
|
927
|
-
if (item.ok === false && isObject(item.error)) {
|
|
928
|
-
const error = item.error;
|
|
929
|
-
return {
|
|
930
|
-
documentId,
|
|
931
|
-
itemError: {
|
|
932
|
-
code: stringField(error, "code") ?? "STATUS_ITEM_ERROR",
|
|
933
|
-
message: stringField(error, "message") ?? "Document status lookup failed.",
|
|
934
|
-
retryable: typeof error.retryable === "boolean" ? error.retryable : false,
|
|
935
|
-
},
|
|
936
|
-
raw: item,
|
|
937
|
-
};
|
|
938
|
-
}
|
|
939
|
-
return statusItemFromPayload(item, documentId);
|
|
940
|
-
}
|
|
941
774
|
async function initializeBulkJob(input) {
|
|
942
775
|
await mkdir(dirname(input.statePath), { recursive: true });
|
|
943
776
|
const db = await openSqlite(input.statePath);
|
|
@@ -2271,13 +2104,6 @@ async function loadOptionalSchemaSnapshot(args, env) {
|
|
|
2271
2104
|
throw error;
|
|
2272
2105
|
}
|
|
2273
2106
|
}
|
|
2274
|
-
async function resolveCollection(config, selector, options) {
|
|
2275
|
-
const params = new URLSearchParams({ action: "upload" });
|
|
2276
|
-
params.set(selector.field, selector.value);
|
|
2277
|
-
if (options.includeSchema)
|
|
2278
|
-
params.set("include_schema", "true");
|
|
2279
|
-
return jsonRequest(config, `collections/resolve?${params.toString()}`);
|
|
2280
|
-
}
|
|
2281
2107
|
async function loadMetadataMap(args) {
|
|
2282
2108
|
const metadataMapFile = getString(args, "metadata-map");
|
|
2283
2109
|
if (!metadataMapFile) {
|
|
@@ -2724,93 +2550,6 @@ async function uploadOne(input) {
|
|
|
2724
2550
|
},
|
|
2725
2551
|
});
|
|
2726
2552
|
}
|
|
2727
|
-
async function resolveSelectorFields(config, selector) {
|
|
2728
|
-
if (selector.field !== "collection_name") {
|
|
2729
|
-
return { [selector.field]: selector.value };
|
|
2730
|
-
}
|
|
2731
|
-
const matches = (await listCollections(config, "upload", 100)).filter((item) => item.name === selector.value);
|
|
2732
|
-
if (matches.length === 0)
|
|
2733
|
-
throw new CliError(`No uploadable collection matched name: ${selector.value}`);
|
|
2734
|
-
if (matches.length > 1) {
|
|
2735
|
-
const choices = matches.map((item) => item.key ?? item.path ?? item.id ?? "").join(", ");
|
|
2736
|
-
throw new CliError(`Collection name is not unique: ${selector.value}. Use --collection-key or --collection-path. Matches: ${choices}`);
|
|
2737
|
-
}
|
|
2738
|
-
const match = matches[0];
|
|
2739
|
-
const key = collectionKey(match);
|
|
2740
|
-
if (key)
|
|
2741
|
-
return { collection_key: key };
|
|
2742
|
-
const path = collectionPath(match);
|
|
2743
|
-
if (path)
|
|
2744
|
-
return { collection_path: path };
|
|
2745
|
-
if (typeof match.id === "string" && match.id)
|
|
2746
|
-
return { primary_collection_id: match.id };
|
|
2747
|
-
throw new CliError(`Collection matched name ${selector.value}, but response had no key, path, or id.`);
|
|
2748
|
-
}
|
|
2749
|
-
async function listCollections(config, capability, limit) {
|
|
2750
|
-
const collections = [];
|
|
2751
|
-
let offset = 0;
|
|
2752
|
-
while (true) {
|
|
2753
|
-
const payload = await jsonRequest(config, `collections?${new URLSearchParams({
|
|
2754
|
-
capability,
|
|
2755
|
-
limit: String(limit),
|
|
2756
|
-
offset: String(offset),
|
|
2757
|
-
}).toString()}`);
|
|
2758
|
-
const page = collectionItems(payload);
|
|
2759
|
-
collections.push(...page);
|
|
2760
|
-
if (page.length < limit)
|
|
2761
|
-
return collections;
|
|
2762
|
-
offset += limit;
|
|
2763
|
-
}
|
|
2764
|
-
}
|
|
2765
|
-
async function jsonRequest(config, path, init = {}) {
|
|
2766
|
-
const url = `${config.apiBaseUrl}${config.apiPathPrefix}/${path.replace(/^\/+/, "")}`;
|
|
2767
|
-
const headers = new Headers(init.headers);
|
|
2768
|
-
headers.set("Authorization", `Bearer ${config.apiKey}`);
|
|
2769
|
-
headers.set("Accept", "application/json");
|
|
2770
|
-
if (init.body && !(init.body instanceof FormData)) {
|
|
2771
|
-
headers.set("Content-Type", "application/json");
|
|
2772
|
-
}
|
|
2773
|
-
let response;
|
|
2774
|
-
try {
|
|
2775
|
-
response = await fetch(url, {
|
|
2776
|
-
...init,
|
|
2777
|
-
headers,
|
|
2778
|
-
signal: AbortSignal.timeout(config.timeoutSeconds * 1000),
|
|
2779
|
-
});
|
|
2780
|
-
}
|
|
2781
|
-
catch (error) {
|
|
2782
|
-
throw new HttpError(`Request failed for ${url}: ${error instanceof Error ? error.message : String(error)}`, undefined, undefined, true);
|
|
2783
|
-
}
|
|
2784
|
-
const text = await response.text();
|
|
2785
|
-
if (!response.ok) {
|
|
2786
|
-
throw new HttpError(`HTTP ${response.status} from ${url}: ${text}`, response.status, retryAfterSeconds(response.headers), RETRYABLE_HTTP_STATUSES.has(response.status));
|
|
2787
|
-
}
|
|
2788
|
-
if (!text.trim())
|
|
2789
|
-
return {};
|
|
2790
|
-
try {
|
|
2791
|
-
return JSON.parse(text);
|
|
2792
|
-
}
|
|
2793
|
-
catch {
|
|
2794
|
-
throw new CliError(`Expected JSON from ${url}, got: ${text.slice(0, 200)}`);
|
|
2795
|
-
}
|
|
2796
|
-
}
|
|
2797
|
-
function resolveConfig(args, env, options = {}) {
|
|
2798
|
-
const requireKey = options.requireKey ?? true;
|
|
2799
|
-
const apiKey = getString(args, "api-key") ?? firstEnv(env, "TIANGONG_AI_API_KEY", "TIANGONG_KB_API_KEY") ?? "";
|
|
2800
|
-
if (requireKey && !apiKey) {
|
|
2801
|
-
throw new CliError("Missing API key. Provide --api-key or set TIANGONG_AI_API_KEY.");
|
|
2802
|
-
}
|
|
2803
|
-
return {
|
|
2804
|
-
apiBaseUrl: (getString(args, "api-base-url") ??
|
|
2805
|
-
firstEnv(env, "TIANGONG_KB_API_BASE_URL") ??
|
|
2806
|
-
DEFAULT_API_BASE_URL).replace(/\/+$/, ""),
|
|
2807
|
-
apiPathPrefix: normalizePrefix(getString(args, "api-path-prefix") ??
|
|
2808
|
-
firstEnv(env, "TIANGONG_KB_API_PATH_PREFIX") ??
|
|
2809
|
-
DEFAULT_API_PATH_PREFIX),
|
|
2810
|
-
apiKey,
|
|
2811
|
-
timeoutSeconds: positiveNumberValue(getString(args, "timeout") ?? firstEnv(env, "TIANGONG_KB_TIMEOUT"), 120, "--timeout"),
|
|
2812
|
-
};
|
|
2813
|
-
}
|
|
2814
2553
|
async function collectFiles(path, recursive, options = {}) {
|
|
2815
2554
|
const item = await stat(path).catch(() => undefined);
|
|
2816
2555
|
if (!item)
|
|
@@ -2873,36 +2612,6 @@ async function runPool(items, concurrency, worker) {
|
|
|
2873
2612
|
await Promise.all(workers);
|
|
2874
2613
|
return results;
|
|
2875
2614
|
}
|
|
2876
|
-
function collectionItems(payload) {
|
|
2877
|
-
const data = responseData(payload);
|
|
2878
|
-
if (Array.isArray(data))
|
|
2879
|
-
return data.filter(isObject);
|
|
2880
|
-
if (isObject(data) && Array.isArray(data.collections))
|
|
2881
|
-
return data.collections.filter(isObject);
|
|
2882
|
-
if (isObject(data) && Array.isArray(data.data))
|
|
2883
|
-
return data.data.filter(isObject);
|
|
2884
|
-
if (isObject(payload) && Array.isArray(payload.data))
|
|
2885
|
-
return payload.data.filter(isObject);
|
|
2886
|
-
throw new CliError("Collection list response did not contain a data array.");
|
|
2887
|
-
}
|
|
2888
|
-
function collectionKey(item) {
|
|
2889
|
-
return (stringField(item, "key") ??
|
|
2890
|
-
stringField(item, "collectionKey") ??
|
|
2891
|
-
stringField(item, "collection_key"));
|
|
2892
|
-
}
|
|
2893
|
-
function collectionPath(item) {
|
|
2894
|
-
return (stringField(item, "path") ??
|
|
2895
|
-
stringField(item, "collectionPath") ??
|
|
2896
|
-
stringField(item, "collection_path"));
|
|
2897
|
-
}
|
|
2898
|
-
function responseData(payload) {
|
|
2899
|
-
if (isObject(payload) &&
|
|
2900
|
-
"data" in payload &&
|
|
2901
|
-
("api_version" in payload || "request_id" in payload || isObject(payload.data))) {
|
|
2902
|
-
return payload.data;
|
|
2903
|
-
}
|
|
2904
|
-
return payload;
|
|
2905
|
-
}
|
|
2906
2615
|
function documentIdFromUpload(payload) {
|
|
2907
2616
|
const data = responseData(payload);
|
|
2908
2617
|
if (!isObject(data))
|
|
@@ -2934,70 +2643,6 @@ function formatUploadLine(result) {
|
|
|
2934
2643
|
const existing = result.existingDocumentId ? ` existing=${result.existingDocumentId}` : "";
|
|
2935
2644
|
return `OK ${result.path} document=${result.documentId ?? ""}${duplicate}${existing}\n`;
|
|
2936
2645
|
}
|
|
2937
|
-
function addSelector(selectors, field, value) {
|
|
2938
|
-
if (value)
|
|
2939
|
-
selectors.push({ field, value });
|
|
2940
|
-
}
|
|
2941
|
-
function getString(args, name) {
|
|
2942
|
-
const value = args.flags.get(name);
|
|
2943
|
-
if (typeof value !== "string" || value.trim() === "")
|
|
2944
|
-
return undefined;
|
|
2945
|
-
return value.trim();
|
|
2946
|
-
}
|
|
2947
|
-
function getBoolean(args, name) {
|
|
2948
|
-
return args.flags.get(name) === true || args.flags.get(name) === "true";
|
|
2949
|
-
}
|
|
2950
|
-
function getPositiveInteger(args, name, defaultValue) {
|
|
2951
|
-
return positiveIntegerValue(getString(args, name), defaultValue, `--${name}`);
|
|
2952
|
-
}
|
|
2953
|
-
function positiveIntegerValue(value, defaultValue, label) {
|
|
2954
|
-
if (!value)
|
|
2955
|
-
return defaultValue;
|
|
2956
|
-
const parsed = Number(value);
|
|
2957
|
-
if (!Number.isInteger(parsed) || parsed <= 0)
|
|
2958
|
-
throw new CliError(`${label} must be a positive integer.`);
|
|
2959
|
-
return parsed;
|
|
2960
|
-
}
|
|
2961
|
-
function getNonNegativeInteger(args, name, defaultValue) {
|
|
2962
|
-
return nonNegativeIntegerValue(getString(args, name), defaultValue, `--${name}`);
|
|
2963
|
-
}
|
|
2964
|
-
function nonNegativeIntegerValue(value, defaultValue, label) {
|
|
2965
|
-
if (!value)
|
|
2966
|
-
return defaultValue;
|
|
2967
|
-
const parsed = Number(value);
|
|
2968
|
-
if (!Number.isInteger(parsed) || parsed < 0)
|
|
2969
|
-
throw new CliError(`${label} must be a non-negative integer.`);
|
|
2970
|
-
return parsed;
|
|
2971
|
-
}
|
|
2972
|
-
function getPositiveNumber(args, name, defaultValue) {
|
|
2973
|
-
return positiveNumberValue(getString(args, name), defaultValue, `--${name}`);
|
|
2974
|
-
}
|
|
2975
|
-
function positiveNumberValue(value, defaultValue, label) {
|
|
2976
|
-
if (!value)
|
|
2977
|
-
return defaultValue;
|
|
2978
|
-
const parsed = Number(value);
|
|
2979
|
-
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
2980
|
-
throw new CliError(`${label} must be a positive number.`);
|
|
2981
|
-
return parsed;
|
|
2982
|
-
}
|
|
2983
|
-
function firstEnv(env, ...names) {
|
|
2984
|
-
for (const name of names) {
|
|
2985
|
-
const value = env[name]?.trim();
|
|
2986
|
-
if (value)
|
|
2987
|
-
return value;
|
|
2988
|
-
}
|
|
2989
|
-
return undefined;
|
|
2990
|
-
}
|
|
2991
|
-
function normalizePrefix(value) {
|
|
2992
|
-
return `/${value.replace(/^\/+|\/+$/g, "")}`;
|
|
2993
|
-
}
|
|
2994
|
-
function retryAfterSeconds(headers) {
|
|
2995
|
-
const value = headers.get("Retry-After");
|
|
2996
|
-
if (!value)
|
|
2997
|
-
return undefined;
|
|
2998
|
-
const parsed = Number(value);
|
|
2999
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
3000
|
-
}
|
|
3001
2646
|
function retryDelay(error, attempt) {
|
|
3002
2647
|
if (error instanceof HttpError && error.retryAfterSeconds)
|
|
3003
2648
|
return Math.min(error.retryAfterSeconds, 30);
|
|
@@ -3023,49 +2668,6 @@ function mimeType(path) {
|
|
|
3023
2668
|
}
|
|
3024
2669
|
return "application/octet-stream";
|
|
3025
2670
|
}
|
|
3026
|
-
function isUuid(value) {
|
|
3027
|
-
return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value);
|
|
3028
|
-
}
|
|
3029
|
-
function isObject(value) {
|
|
3030
|
-
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
3031
|
-
}
|
|
3032
|
-
function stringField(value, field) {
|
|
3033
|
-
const candidate = value[field];
|
|
3034
|
-
return typeof candidate === "string" && candidate ? candidate : undefined;
|
|
3035
|
-
}
|
|
3036
|
-
function write(output, text) {
|
|
3037
|
-
output.write(text);
|
|
3038
|
-
}
|
|
3039
|
-
function loadDotenv(env) {
|
|
3040
|
-
const cwd = dirname(fileURLToPath(import.meta.url));
|
|
3041
|
-
const candidates = [resolve(process.cwd(), ".env"), resolve(cwd, "../.env")];
|
|
3042
|
-
for (const candidate of candidates) {
|
|
3043
|
-
const content = readEnvFile(candidate);
|
|
3044
|
-
if (!content)
|
|
3045
|
-
continue;
|
|
3046
|
-
for (const line of content.split(/\r?\n/)) {
|
|
3047
|
-
const trimmed = line.trim();
|
|
3048
|
-
if (!trimmed || trimmed.startsWith("#") || !trimmed.includes("="))
|
|
3049
|
-
continue;
|
|
3050
|
-
const [rawKey, ...rawValue] = trimmed.replace(/^export\s+/, "").split("=");
|
|
3051
|
-
const key = rawKey?.trim();
|
|
3052
|
-
if (!key || env[key])
|
|
3053
|
-
continue;
|
|
3054
|
-
env[key] = rawValue
|
|
3055
|
-
.join("=")
|
|
3056
|
-
.trim()
|
|
3057
|
-
.replace(/^["']|["']$/g, "");
|
|
3058
|
-
}
|
|
3059
|
-
}
|
|
3060
|
-
}
|
|
3061
|
-
function readEnvFile(path) {
|
|
3062
|
-
try {
|
|
3063
|
-
return readFileSync(path, "utf8");
|
|
3064
|
-
}
|
|
3065
|
-
catch {
|
|
3066
|
-
return undefined;
|
|
3067
|
-
}
|
|
3068
|
-
}
|
|
3069
2671
|
function topHelp() {
|
|
3070
2672
|
return `Tiangong AI CLI
|
|
3071
2673
|
|
|
@@ -3075,8 +2677,12 @@ Usage:
|
|
|
3075
2677
|
tiangong-ai kb ingest jobs
|
|
3076
2678
|
tiangong-ai kb ingest status <document-id>
|
|
3077
2679
|
tiangong-ai kb collections list [--capability upload]
|
|
2680
|
+
tiangong-ai research search --input <request.json>|--query <query> [--sources default|all|sci|report|patent]
|
|
2681
|
+
tiangong-ai education search --input <request.json>|--query <query> [--sources default|all|course|edu|textbook]
|
|
3078
2682
|
|
|
3079
2683
|
Run "tiangong-ai kb --help" for KB options.
|
|
2684
|
+
Run "tiangong-ai research --help" for research options.
|
|
2685
|
+
Run "tiangong-ai education --help" for education options.
|
|
3080
2686
|
`;
|
|
3081
2687
|
}
|
|
3082
2688
|
function kbHelp() {
|