@arbidocs/cli 0.3.28 → 0.3.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +114 -1
- package/dist/index.js +1002 -134
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
4
|
var commander = require('commander');
|
|
5
|
-
var
|
|
6
|
-
var
|
|
5
|
+
var fs4 = require('fs');
|
|
6
|
+
var path4 = require('path');
|
|
7
7
|
var os = require('os');
|
|
8
8
|
var chalk2 = require('chalk');
|
|
9
9
|
var sdk = require('@arbidocs/sdk');
|
|
@@ -16,24 +16,24 @@ var module$1 = require('module');
|
|
|
16
16
|
var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
|
|
17
17
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
18
18
|
|
|
19
|
-
var
|
|
20
|
-
var
|
|
19
|
+
var fs4__default = /*#__PURE__*/_interopDefault(fs4);
|
|
20
|
+
var path4__default = /*#__PURE__*/_interopDefault(path4);
|
|
21
21
|
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
22
22
|
var chalk2__default = /*#__PURE__*/_interopDefault(chalk2);
|
|
23
23
|
|
|
24
24
|
function getCacheFile() {
|
|
25
|
-
const configDir = process.env.ARBI_CONFIG_DIR ??
|
|
26
|
-
return
|
|
25
|
+
const configDir = process.env.ARBI_CONFIG_DIR ?? path4__default.default.join(os__default.default.homedir(), ".arbi");
|
|
26
|
+
return path4__default.default.join(configDir, "completions.json");
|
|
27
27
|
}
|
|
28
28
|
function ensureDir(filePath) {
|
|
29
|
-
const dir =
|
|
30
|
-
if (!
|
|
31
|
-
|
|
29
|
+
const dir = path4__default.default.dirname(filePath);
|
|
30
|
+
if (!fs4__default.default.existsSync(dir)) {
|
|
31
|
+
fs4__default.default.mkdirSync(dir, { recursive: true, mode: 448 });
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
34
|
function getCachedWorkspaceIds() {
|
|
35
35
|
try {
|
|
36
|
-
const content =
|
|
36
|
+
const content = fs4__default.default.readFileSync(getCacheFile(), "utf-8");
|
|
37
37
|
const cache = JSON.parse(content);
|
|
38
38
|
return cache.workspaces.map((w) => w.id);
|
|
39
39
|
} catch {
|
|
@@ -47,7 +47,7 @@ function updateCompletionCache(workspaces3) {
|
|
|
47
47
|
};
|
|
48
48
|
const filePath = getCacheFile();
|
|
49
49
|
ensureDir(filePath);
|
|
50
|
-
|
|
50
|
+
fs4__default.default.writeFileSync(filePath, JSON.stringify(cache, null, 2) + "\n", { mode: 384 });
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
// src/completion.ts
|
|
@@ -194,12 +194,12 @@ var ALIAS_LINE = 'alias A="arbi ask"';
|
|
|
194
194
|
var ALIAS_MARKER = "# arbi-cli alias";
|
|
195
195
|
function getShellRcPath() {
|
|
196
196
|
const shell = process.env.SHELL || "";
|
|
197
|
-
if (shell.includes("zsh")) return
|
|
198
|
-
return
|
|
197
|
+
if (shell.includes("zsh")) return path4.join(os.homedir(), ".zshrc");
|
|
198
|
+
return path4.join(os.homedir(), ".bashrc");
|
|
199
199
|
}
|
|
200
200
|
function isAliasInstalled(rcPath) {
|
|
201
|
-
if (!
|
|
202
|
-
const content =
|
|
201
|
+
if (!fs4.existsSync(rcPath)) return false;
|
|
202
|
+
const content = fs4.readFileSync(rcPath, "utf-8");
|
|
203
203
|
return content.includes(ALIAS_LINE) || content.includes(ALIAS_MARKER);
|
|
204
204
|
}
|
|
205
205
|
function registerConfigCommand(program2) {
|
|
@@ -278,7 +278,7 @@ function registerConfigCommand(program2) {
|
|
|
278
278
|
dim("Usage: A what is the meaning of life");
|
|
279
279
|
return;
|
|
280
280
|
}
|
|
281
|
-
|
|
281
|
+
fs4.appendFileSync(rcPath, `
|
|
282
282
|
${ALIAS_MARKER}
|
|
283
283
|
${ALIAS_LINE}
|
|
284
284
|
`);
|
|
@@ -3600,11 +3600,11 @@ async function promptPassword(message) {
|
|
|
3600
3600
|
async function promptConfirm(message, defaultValue = true) {
|
|
3601
3601
|
return prompts.confirm({ message, default: defaultValue });
|
|
3602
3602
|
}
|
|
3603
|
-
var CACHE_FILE =
|
|
3603
|
+
var CACHE_FILE = path4__default.default.join(os__default.default.homedir(), ".arbi", "version-cache.json");
|
|
3604
3604
|
var CACHE_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
3605
3605
|
function readCache() {
|
|
3606
3606
|
try {
|
|
3607
|
-
const data = JSON.parse(
|
|
3607
|
+
const data = JSON.parse(fs4__default.default.readFileSync(CACHE_FILE, "utf8"));
|
|
3608
3608
|
if (data.latest && data.checkedAt && Date.now() - data.checkedAt < CACHE_TTL_MS) {
|
|
3609
3609
|
return data;
|
|
3610
3610
|
}
|
|
@@ -3614,9 +3614,9 @@ function readCache() {
|
|
|
3614
3614
|
}
|
|
3615
3615
|
function writeCache(latest) {
|
|
3616
3616
|
try {
|
|
3617
|
-
const dir =
|
|
3618
|
-
if (!
|
|
3619
|
-
|
|
3617
|
+
const dir = path4__default.default.dirname(CACHE_FILE);
|
|
3618
|
+
if (!fs4__default.default.existsSync(dir)) fs4__default.default.mkdirSync(dir, { recursive: true, mode: 448 });
|
|
3619
|
+
fs4__default.default.writeFileSync(CACHE_FILE, JSON.stringify({ latest, checkedAt: Date.now() }) + "\n");
|
|
3620
3620
|
} catch {
|
|
3621
3621
|
}
|
|
3622
3622
|
}
|
|
@@ -3637,13 +3637,13 @@ function getLatestVersion(skipCache = false) {
|
|
|
3637
3637
|
}
|
|
3638
3638
|
}
|
|
3639
3639
|
function getCurrentVersion() {
|
|
3640
|
-
return "0.3.
|
|
3640
|
+
return "0.3.30";
|
|
3641
3641
|
}
|
|
3642
3642
|
function readChangelog(fromVersion, toVersion) {
|
|
3643
3643
|
try {
|
|
3644
3644
|
const globalRoot = child_process.execSync("npm root -g", { encoding: "utf8", timeout: 5e3 }).trim();
|
|
3645
|
-
const changelogPath =
|
|
3646
|
-
const text =
|
|
3645
|
+
const changelogPath = path4__default.default.join(globalRoot, "@arbidocs", "cli", "CHANGELOG.md");
|
|
3646
|
+
const text = fs4__default.default.readFileSync(changelogPath, "utf8");
|
|
3647
3647
|
return extractSections(text, fromVersion, toVersion);
|
|
3648
3648
|
} catch {
|
|
3649
3649
|
return null;
|
|
@@ -3690,17 +3690,17 @@ function showChangelog(fromVersion, toVersion) {
|
|
|
3690
3690
|
async function checkForUpdates(autoUpdate) {
|
|
3691
3691
|
try {
|
|
3692
3692
|
const latest = getLatestVersion();
|
|
3693
|
-
if (!latest || latest === "0.3.
|
|
3693
|
+
if (!latest || latest === "0.3.30") return;
|
|
3694
3694
|
if (autoUpdate) {
|
|
3695
3695
|
warn(`
|
|
3696
|
-
Your arbi version is out of date (${"0.3.
|
|
3696
|
+
Your arbi version is out of date (${"0.3.30"} \u2192 ${latest}). Updating...`);
|
|
3697
3697
|
child_process.execSync("npm install -g @arbidocs/cli@latest", { stdio: "inherit" });
|
|
3698
|
-
showChangelog("0.3.
|
|
3698
|
+
showChangelog("0.3.30", latest);
|
|
3699
3699
|
console.log(`Updated to ${latest}.`);
|
|
3700
3700
|
} else {
|
|
3701
3701
|
warn(
|
|
3702
3702
|
`
|
|
3703
|
-
Your arbi version is out of date (${"0.3.
|
|
3703
|
+
Your arbi version is out of date (${"0.3.30"} \u2192 ${latest}).
|
|
3704
3704
|
Run "arbi update" to upgrade, or "arbi update auto" to always stay up to date.`
|
|
3705
3705
|
);
|
|
3706
3706
|
}
|
|
@@ -3710,9 +3710,9 @@ Run "arbi update" to upgrade, or "arbi update auto" to always stay up to date.`
|
|
|
3710
3710
|
function hintUpdateOnError() {
|
|
3711
3711
|
try {
|
|
3712
3712
|
const cached = readCache();
|
|
3713
|
-
if (cached && cached.latest !== "0.3.
|
|
3713
|
+
if (cached && cached.latest !== "0.3.30") {
|
|
3714
3714
|
warn(
|
|
3715
|
-
`Your arbi version is out of date (${"0.3.
|
|
3715
|
+
`Your arbi version is out of date (${"0.3.30"} \u2192 ${cached.latest}). Run "arbi update".`
|
|
3716
3716
|
);
|
|
3717
3717
|
}
|
|
3718
3718
|
} catch {
|
|
@@ -3721,18 +3721,18 @@ function hintUpdateOnError() {
|
|
|
3721
3721
|
var MAX_TASKS = 50;
|
|
3722
3722
|
var MAX_AGE_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
3723
3723
|
function getTasksFile() {
|
|
3724
|
-
const configDir = process.env.ARBI_CONFIG_DIR ??
|
|
3725
|
-
return
|
|
3724
|
+
const configDir = process.env.ARBI_CONFIG_DIR ?? path4__default.default.join(os__default.default.homedir(), ".arbi");
|
|
3725
|
+
return path4__default.default.join(configDir, "tasks.json");
|
|
3726
3726
|
}
|
|
3727
3727
|
function ensureDir2(filePath) {
|
|
3728
|
-
const dir =
|
|
3729
|
-
if (!
|
|
3730
|
-
|
|
3728
|
+
const dir = path4__default.default.dirname(filePath);
|
|
3729
|
+
if (!fs4__default.default.existsSync(dir)) {
|
|
3730
|
+
fs4__default.default.mkdirSync(dir, { recursive: true, mode: 448 });
|
|
3731
3731
|
}
|
|
3732
3732
|
}
|
|
3733
3733
|
function readTasks() {
|
|
3734
3734
|
try {
|
|
3735
|
-
const content =
|
|
3735
|
+
const content = fs4__default.default.readFileSync(getTasksFile(), "utf-8");
|
|
3736
3736
|
return JSON.parse(content);
|
|
3737
3737
|
} catch {
|
|
3738
3738
|
return [];
|
|
@@ -3741,7 +3741,7 @@ function readTasks() {
|
|
|
3741
3741
|
function writeTasks(tasks) {
|
|
3742
3742
|
const filePath = getTasksFile();
|
|
3743
3743
|
ensureDir2(filePath);
|
|
3744
|
-
|
|
3744
|
+
fs4__default.default.writeFileSync(filePath, JSON.stringify(tasks, null, 2) + "\n", { mode: 384 });
|
|
3745
3745
|
}
|
|
3746
3746
|
function getTasks() {
|
|
3747
3747
|
const now = Date.now();
|
|
@@ -3886,11 +3886,11 @@ async function resolveAuth() {
|
|
|
3886
3886
|
throw err;
|
|
3887
3887
|
}
|
|
3888
3888
|
}
|
|
3889
|
-
async function resolveWorkspace(workspaceOpt) {
|
|
3889
|
+
async function resolveWorkspace(workspaceOpt, { skipNotifications = false } = {}) {
|
|
3890
3890
|
try {
|
|
3891
3891
|
resolveConfig();
|
|
3892
3892
|
const ctx = await sdk.resolveWorkspace(store, workspaceOpt);
|
|
3893
|
-
if (getConfig()?.notifications !== false) {
|
|
3893
|
+
if (!skipNotifications && getConfig()?.notifications !== false) {
|
|
3894
3894
|
startBackgroundNotifications(ctx.config.baseUrl, ctx.accessToken).catch(() => {
|
|
3895
3895
|
});
|
|
3896
3896
|
}
|
|
@@ -3975,39 +3975,39 @@ function checkAgentDependency(backend) {
|
|
|
3975
3975
|
dim(`${binary} ${match?.[1] ?? version} \u2713`);
|
|
3976
3976
|
}
|
|
3977
3977
|
function loadSkillContent() {
|
|
3978
|
-
const cliRoot =
|
|
3978
|
+
const cliRoot = path4.resolve(path4.dirname(url.fileURLToPath((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.js', document.baseURI).href)))), "..");
|
|
3979
3979
|
try {
|
|
3980
|
-
return
|
|
3980
|
+
return fs4.readFileSync(path4.join(cliRoot, "SKILL.md"), "utf-8");
|
|
3981
3981
|
} catch {
|
|
3982
3982
|
return void 0;
|
|
3983
3983
|
}
|
|
3984
3984
|
}
|
|
3985
3985
|
function installSkill(backend) {
|
|
3986
|
-
const cliRoot =
|
|
3986
|
+
const cliRoot = path4.resolve(path4.dirname(url.fileURLToPath((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.js', document.baseURI).href)))), "..");
|
|
3987
3987
|
let skillContent;
|
|
3988
3988
|
try {
|
|
3989
|
-
skillContent =
|
|
3989
|
+
skillContent = fs4.readFileSync(path4.join(cliRoot, "SKILL.md"), "utf-8");
|
|
3990
3990
|
} catch {
|
|
3991
3991
|
return;
|
|
3992
3992
|
}
|
|
3993
3993
|
if (backend === "claude") {
|
|
3994
|
-
const dir =
|
|
3995
|
-
const target =
|
|
3994
|
+
const dir = path4.join(os.homedir(), ".claude", "commands", "arbi");
|
|
3995
|
+
const target = path4.join(dir, "SKILL.md");
|
|
3996
3996
|
try {
|
|
3997
|
-
|
|
3998
|
-
if (!
|
|
3999
|
-
|
|
3997
|
+
fs4.mkdirSync(dir, { recursive: true });
|
|
3998
|
+
if (!fs4.existsSync(target) || fs4.readFileSync(target, "utf-8") !== skillContent) {
|
|
3999
|
+
fs4.writeFileSync(target, skillContent, "utf-8");
|
|
4000
4000
|
dim(`Installed ARBI skill \u2192 ${target}`);
|
|
4001
4001
|
}
|
|
4002
4002
|
} catch {
|
|
4003
4003
|
}
|
|
4004
4004
|
} else if (backend === "openclaw") {
|
|
4005
|
-
const workspace =
|
|
4006
|
-
const bootstrap =
|
|
4005
|
+
const workspace = path4.join(os.homedir(), ".arbi", "openclaw-workspace");
|
|
4006
|
+
const bootstrap = path4.join(workspace, "BOOTSTRAP.md");
|
|
4007
4007
|
try {
|
|
4008
|
-
|
|
4009
|
-
if (!
|
|
4010
|
-
|
|
4008
|
+
fs4.mkdirSync(workspace, { recursive: true });
|
|
4009
|
+
if (!fs4.existsSync(bootstrap) || fs4.readFileSync(bootstrap, "utf-8") !== skillContent) {
|
|
4010
|
+
fs4.writeFileSync(bootstrap, skillContent, "utf-8");
|
|
4011
4011
|
dim(`Installed ARBI skill \u2192 ${bootstrap}`);
|
|
4012
4012
|
}
|
|
4013
4013
|
} catch {
|
|
@@ -4335,8 +4335,16 @@ async function loginAfterRegister(config, email, password2) {
|
|
|
4335
4335
|
const memberWorkspaces = wsList.filter((ws2) => ws2.users?.some((u) => u.user.email === email));
|
|
4336
4336
|
if (memberWorkspaces.length === 0) {
|
|
4337
4337
|
console.log("Creating your first workspace...");
|
|
4338
|
+
const userProjects = await sdk.projects.listProjects(arbi);
|
|
4339
|
+
const defaultProjectExtId = userProjects[0]?.external_id;
|
|
4340
|
+
if (!defaultProjectExtId) throw new Error("No projects found for user");
|
|
4338
4341
|
const encryptedKey = await sdk.generateNewWorkspaceKey(arbi, loginResult.serverSessionKey);
|
|
4339
|
-
const ws2 = await sdk.workspaces.createWorkspace(
|
|
4342
|
+
const ws2 = await sdk.workspaces.createWorkspace(
|
|
4343
|
+
arbi,
|
|
4344
|
+
"My First Workspace",
|
|
4345
|
+
encryptedKey,
|
|
4346
|
+
defaultProjectExtId
|
|
4347
|
+
);
|
|
4340
4348
|
updateConfig({ selectedWorkspaceId: ws2.external_id });
|
|
4341
4349
|
success(`Workspace: ${ws2.name} (${ref(ws2.external_id)})`);
|
|
4342
4350
|
return;
|
|
@@ -4451,11 +4459,15 @@ function registerWorkspacesCommand(program2) {
|
|
|
4451
4459
|
workspace.command("create <name>").description("Create a new workspace").option("-d, --description <text>", "Workspace description").option("--public", "Make workspace public", false).action(
|
|
4452
4460
|
(name, opts) => runAction(async () => {
|
|
4453
4461
|
const { arbi, loginResult } = await resolveAuth();
|
|
4462
|
+
const userProjects = await sdk.projects.listProjects(arbi);
|
|
4463
|
+
if (!userProjects.length) throw new Error("No projects found. Create a project first.");
|
|
4464
|
+
const projectExtId = userProjects[0].external_id;
|
|
4454
4465
|
const encryptedKey = await sdk.generateNewWorkspaceKey(arbi, loginResult.serverSessionKey);
|
|
4455
4466
|
const data = await sdk.workspaces.createWorkspace(
|
|
4456
4467
|
arbi,
|
|
4457
4468
|
name,
|
|
4458
4469
|
encryptedKey,
|
|
4470
|
+
projectExtId,
|
|
4459
4471
|
opts.description,
|
|
4460
4472
|
opts.public ?? false
|
|
4461
4473
|
);
|
|
@@ -4588,7 +4600,63 @@ function statusSymbol(status2) {
|
|
|
4588
4600
|
return "\u25CF";
|
|
4589
4601
|
}
|
|
4590
4602
|
}
|
|
4591
|
-
|
|
4603
|
+
function statusColor(status2) {
|
|
4604
|
+
switch (status2) {
|
|
4605
|
+
case "completed":
|
|
4606
|
+
return chalk2__default.default.green;
|
|
4607
|
+
case "failed":
|
|
4608
|
+
return chalk2__default.default.red;
|
|
4609
|
+
case "queued":
|
|
4610
|
+
return chalk2__default.default.yellow;
|
|
4611
|
+
case "processing":
|
|
4612
|
+
case "parsing":
|
|
4613
|
+
case "indexing":
|
|
4614
|
+
case "analysing":
|
|
4615
|
+
case "encrypting":
|
|
4616
|
+
return chalk2__default.default.cyan;
|
|
4617
|
+
default:
|
|
4618
|
+
return (s) => s;
|
|
4619
|
+
}
|
|
4620
|
+
}
|
|
4621
|
+
function formatSize(bytes) {
|
|
4622
|
+
if (!bytes) return "-";
|
|
4623
|
+
if (bytes < 1024) return `${bytes}B`;
|
|
4624
|
+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}K`;
|
|
4625
|
+
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)}M`;
|
|
4626
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)}G`;
|
|
4627
|
+
}
|
|
4628
|
+
function filterDocs(data, opts) {
|
|
4629
|
+
let result = data;
|
|
4630
|
+
if (opts.status) {
|
|
4631
|
+
const statuses = opts.status.split(",");
|
|
4632
|
+
result = result.filter((d) => statuses.includes(d.status));
|
|
4633
|
+
}
|
|
4634
|
+
if (opts.folder) {
|
|
4635
|
+
const pattern = opts.folder.toLowerCase();
|
|
4636
|
+
result = result.filter((d) => (d.folder ?? "").toLowerCase().includes(pattern));
|
|
4637
|
+
}
|
|
4638
|
+
if (opts.ext) {
|
|
4639
|
+
const exts = opts.ext.split(",").map((e) => e.startsWith(".") ? e.toLowerCase() : `.${e.toLowerCase()}`);
|
|
4640
|
+
result = result.filter((d) => {
|
|
4641
|
+
const fn = d.file_name ?? "";
|
|
4642
|
+
const dotIdx = fn.lastIndexOf(".");
|
|
4643
|
+
const ext = dotIdx >= 0 ? fn.slice(dotIdx).toLowerCase() : "";
|
|
4644
|
+
return exts.includes(ext);
|
|
4645
|
+
});
|
|
4646
|
+
}
|
|
4647
|
+
if (opts.query) {
|
|
4648
|
+
const q = opts.query.toLowerCase();
|
|
4649
|
+
result = result.filter((d) => {
|
|
4650
|
+
const fn = (d.file_name ?? "").toLowerCase();
|
|
4651
|
+
const meta = d.doc_metadata;
|
|
4652
|
+
const title = (meta?.title ?? "").toLowerCase();
|
|
4653
|
+
const subject = (meta?.doc_subject ?? "").toLowerCase();
|
|
4654
|
+
return fn.includes(q) || title.includes(q) || subject.includes(q);
|
|
4655
|
+
});
|
|
4656
|
+
}
|
|
4657
|
+
return result;
|
|
4658
|
+
}
|
|
4659
|
+
async function fetchDocChoices(arbi, _workspaceId) {
|
|
4592
4660
|
const data = await sdk.documents.listDocuments(arbi);
|
|
4593
4661
|
if (data.length === 0) {
|
|
4594
4662
|
console.log("No documents found.");
|
|
@@ -4601,18 +4669,102 @@ async function fetchDocChoices(arbi, workspaceId) {
|
|
|
4601
4669
|
}));
|
|
4602
4670
|
}
|
|
4603
4671
|
function registerDocsCommand(program2) {
|
|
4604
|
-
program2.command("docs").description("List documents in the active workspace").option("-w, --workspace <id>", "Workspace ID (defaults to selected workspace)").action(
|
|
4672
|
+
program2.command("docs").description("List documents in the active workspace").option("-w, --workspace <id>", "Workspace ID (defaults to selected workspace)").option("--json", "Output as JSON").option("--csv", "Output as CSV").option("--ids", "Output only document IDs (one per line)").option("--count", "Output only the count (combine with --status for filtered count)").option("-s, --status <status>", "Filter by status (comma-separated: completed,failed,queued)").option("-f, --folder <pattern>", "Filter by folder (substring match)").option("-e, --ext <extensions>", "Filter by file extension (comma-separated: pdf,docx,eml)").option("-q, --query <text>", "Search file name, title, or summary").option("--sort <field>", "Sort by field (name, status, date, size, created)", "status").option("-n, --limit <n>", "Limit number of results").action(
|
|
4605
4673
|
(opts) => runAction(async () => {
|
|
4606
|
-
const { arbi
|
|
4607
|
-
|
|
4674
|
+
const { arbi } = await resolveWorkspace(opts.workspace);
|
|
4675
|
+
let data = await sdk.documents.listDocuments(arbi);
|
|
4608
4676
|
if (data.length === 0) {
|
|
4609
4677
|
console.log("No documents found.");
|
|
4610
4678
|
return;
|
|
4611
4679
|
}
|
|
4680
|
+
data = filterDocs(data, opts);
|
|
4681
|
+
const sortField = opts.sort ?? "status";
|
|
4682
|
+
data.sort((a, b) => {
|
|
4683
|
+
switch (sortField) {
|
|
4684
|
+
case "name":
|
|
4685
|
+
return (a.file_name ?? "").localeCompare(b.file_name ?? "");
|
|
4686
|
+
case "size":
|
|
4687
|
+
return (b.file_size ?? 0) - (a.file_size ?? 0);
|
|
4688
|
+
case "date":
|
|
4689
|
+
case "created":
|
|
4690
|
+
return (b.created_at ?? "").localeCompare(a.created_at ?? "");
|
|
4691
|
+
case "status":
|
|
4692
|
+
default:
|
|
4693
|
+
return (a.status ?? "").localeCompare(b.status ?? "");
|
|
4694
|
+
}
|
|
4695
|
+
});
|
|
4696
|
+
if (opts.limit) {
|
|
4697
|
+
data = data.slice(0, parseInt(opts.limit, 10));
|
|
4698
|
+
}
|
|
4699
|
+
if (opts.count) {
|
|
4700
|
+
if (opts.status || opts.folder || opts.ext || opts.query) {
|
|
4701
|
+
console.log(String(data.length));
|
|
4702
|
+
} else {
|
|
4703
|
+
const counts = {};
|
|
4704
|
+
data.forEach((d) => {
|
|
4705
|
+
const s = d.status ?? "unknown";
|
|
4706
|
+
counts[s] = (counts[s] ?? 0) + 1;
|
|
4707
|
+
});
|
|
4708
|
+
const withSummary = data.filter((d) => {
|
|
4709
|
+
const m = d.doc_metadata;
|
|
4710
|
+
return m?.doc_subject;
|
|
4711
|
+
}).length;
|
|
4712
|
+
const totalPages = data.reduce((sum, d) => sum + (d.n_pages ?? 0), 0);
|
|
4713
|
+
const totalTokens = data.reduce((sum, d) => sum + (d.tokens ?? 0), 0);
|
|
4714
|
+
console.log(chalk2__default.default.bold(`Total: ${data.length} documents`));
|
|
4715
|
+
for (const [status2, count] of Object.entries(counts).sort()) {
|
|
4716
|
+
const colorFn = statusColor(status2);
|
|
4717
|
+
const pct = (count / data.length * 100).toFixed(1);
|
|
4718
|
+
console.log(` ${colorFn(statusSymbol(status2))} ${status2}: ${count} (${pct}%)`);
|
|
4719
|
+
}
|
|
4720
|
+
console.log(` With summaries: ${withSummary}`);
|
|
4721
|
+
console.log(` Total pages: ${totalPages.toLocaleString()}`);
|
|
4722
|
+
console.log(` Total tokens: ${totalTokens.toLocaleString()}`);
|
|
4723
|
+
}
|
|
4724
|
+
return;
|
|
4725
|
+
}
|
|
4726
|
+
if (opts.ids) {
|
|
4727
|
+
data.forEach((d) => console.log(d.external_id));
|
|
4728
|
+
return;
|
|
4729
|
+
}
|
|
4730
|
+
if (opts.json) {
|
|
4731
|
+
console.log(JSON.stringify(data));
|
|
4732
|
+
return;
|
|
4733
|
+
}
|
|
4734
|
+
if (opts.csv) {
|
|
4735
|
+
console.log("external_id,status,file_name,file_size,folder,n_pages,tokens,doc_date,title");
|
|
4736
|
+
for (const d of data) {
|
|
4737
|
+
const meta = d.doc_metadata;
|
|
4738
|
+
const csvEscape = (s) => `"${(s ?? "").replace(/"/g, '""')}"`;
|
|
4739
|
+
console.log(
|
|
4740
|
+
[
|
|
4741
|
+
d.external_id,
|
|
4742
|
+
d.status,
|
|
4743
|
+
csvEscape(d.file_name ?? ""),
|
|
4744
|
+
d.file_size ?? "",
|
|
4745
|
+
csvEscape(d.folder ?? ""),
|
|
4746
|
+
d.n_pages ?? "",
|
|
4747
|
+
d.tokens ?? "",
|
|
4748
|
+
meta?.doc_date ?? "",
|
|
4749
|
+
csvEscape(meta?.title ?? "")
|
|
4750
|
+
].join(",")
|
|
4751
|
+
);
|
|
4752
|
+
}
|
|
4753
|
+
return;
|
|
4754
|
+
}
|
|
4755
|
+
console.log(chalk2__default.default.dim(`${data.length} documents
|
|
4756
|
+
`));
|
|
4612
4757
|
printTable(
|
|
4613
4758
|
[
|
|
4614
4759
|
{ header: "ID", width: 24, value: (r) => r.external_id },
|
|
4615
|
-
{
|
|
4760
|
+
{
|
|
4761
|
+
header: "S",
|
|
4762
|
+
width: 4,
|
|
4763
|
+
value: (r) => {
|
|
4764
|
+
const s = r.status;
|
|
4765
|
+
return statusColor(s)(statusSymbol(s));
|
|
4766
|
+
}
|
|
4767
|
+
},
|
|
4616
4768
|
{
|
|
4617
4769
|
header: "TITLE",
|
|
4618
4770
|
width: 34,
|
|
@@ -4621,6 +4773,16 @@ function registerDocsCommand(program2) {
|
|
|
4621
4773
|
return meta?.title ?? r.file_name ?? "Unnamed";
|
|
4622
4774
|
}
|
|
4623
4775
|
},
|
|
4776
|
+
{
|
|
4777
|
+
header: "SIZE",
|
|
4778
|
+
width: 8,
|
|
4779
|
+
value: (r) => formatSize(r.file_size)
|
|
4780
|
+
},
|
|
4781
|
+
{
|
|
4782
|
+
header: "PAGES",
|
|
4783
|
+
width: 6,
|
|
4784
|
+
value: (r) => r.n_pages ? String(r.n_pages) : "-"
|
|
4785
|
+
},
|
|
4624
4786
|
{
|
|
4625
4787
|
header: "DATE",
|
|
4626
4788
|
width: 12,
|
|
@@ -4630,12 +4792,9 @@ function registerDocsCommand(program2) {
|
|
|
4630
4792
|
}
|
|
4631
4793
|
},
|
|
4632
4794
|
{
|
|
4633
|
-
header: "
|
|
4634
|
-
width:
|
|
4635
|
-
value: (r) =>
|
|
4636
|
-
const meta = r.doc_metadata;
|
|
4637
|
-
return meta?.doc_subject ?? "-";
|
|
4638
|
-
}
|
|
4795
|
+
header: "FOLDER",
|
|
4796
|
+
width: 30,
|
|
4797
|
+
value: (r) => r.folder ?? "-"
|
|
4639
4798
|
}
|
|
4640
4799
|
],
|
|
4641
4800
|
data
|
|
@@ -4732,9 +4891,11 @@ function registerDocsCommand(program2) {
|
|
|
4732
4891
|
(urls, opts) => runAction(async () => {
|
|
4733
4892
|
const { arbi, workspaceId } = await resolveWorkspace(opts.workspace);
|
|
4734
4893
|
const data = await sdk.documents.uploadUrl(arbi, urls, workspaceId, opts.shared ?? false);
|
|
4735
|
-
success(`Uploaded: ${data.doc_ext_ids.join(", ")}`);
|
|
4736
|
-
if (data.
|
|
4737
|
-
warn(
|
|
4894
|
+
success(`Uploaded: ${(data.doc_ext_ids ?? []).join(", ")}`);
|
|
4895
|
+
if (data.skipped && data.skipped.length > 0) {
|
|
4896
|
+
warn(
|
|
4897
|
+
`Skipped: ${data.skipped.map((s) => `${s.file_name} (${s.reason})`).join(", ")}`
|
|
4898
|
+
);
|
|
4738
4899
|
}
|
|
4739
4900
|
})()
|
|
4740
4901
|
);
|
|
@@ -4755,68 +4916,262 @@ function registerDocsCommand(program2) {
|
|
|
4755
4916
|
console.log(JSON.stringify(data, null, 2));
|
|
4756
4917
|
})()
|
|
4757
4918
|
);
|
|
4919
|
+
doc.command("reprocess [ids...]").description("Reprocess failed/completed documents (sets status back to processing)").option("-s, --status <status>", "Reprocess all docs with this status (e.g. failed)").option("-f, --folder <pattern>", "Filter by folder (substring match)").option("--dry-run", "Show what would be reprocessed without doing it").option("-b, --batch-size <n>", "Batch size for update requests", "50").action(
|
|
4920
|
+
(ids, opts) => runAction(async () => {
|
|
4921
|
+
const { arbi } = await resolveWorkspace();
|
|
4922
|
+
let docIds;
|
|
4923
|
+
if (ids && ids.length > 0) {
|
|
4924
|
+
docIds = ids;
|
|
4925
|
+
} else if (opts.status) {
|
|
4926
|
+
const allDocs = await sdk.documents.listDocuments(arbi);
|
|
4927
|
+
let filtered = allDocs.filter((d) => d.status === opts.status);
|
|
4928
|
+
if (opts.folder) {
|
|
4929
|
+
const pattern = opts.folder.toLowerCase();
|
|
4930
|
+
filtered = filtered.filter(
|
|
4931
|
+
(d) => (d.folder ?? "").toLowerCase().includes(pattern)
|
|
4932
|
+
);
|
|
4933
|
+
}
|
|
4934
|
+
docIds = filtered.map((d) => d.external_id);
|
|
4935
|
+
} else {
|
|
4936
|
+
error("Provide document IDs or use --status to select documents to reprocess.");
|
|
4937
|
+
process.exit(1);
|
|
4938
|
+
return;
|
|
4939
|
+
}
|
|
4940
|
+
if (docIds.length === 0) {
|
|
4941
|
+
console.log("No documents match the criteria.");
|
|
4942
|
+
return;
|
|
4943
|
+
}
|
|
4944
|
+
console.log(`${docIds.length} document(s) to reprocess.`);
|
|
4945
|
+
if (opts.dryRun) {
|
|
4946
|
+
docIds.forEach((id) => console.log(` ${id}`));
|
|
4947
|
+
console.log(chalk2__default.default.dim("\n(dry run \u2014 no changes made)"));
|
|
4948
|
+
return;
|
|
4949
|
+
}
|
|
4950
|
+
const batchSize = parseInt(opts.batchSize ?? "50", 10);
|
|
4951
|
+
let processed = 0;
|
|
4952
|
+
for (let i = 0; i < docIds.length; i += batchSize) {
|
|
4953
|
+
const batch = docIds.slice(i, i + batchSize);
|
|
4954
|
+
const updates = batch.map((id) => ({ external_id: id, status: "processing" }));
|
|
4955
|
+
await sdk.documents.updateDocuments(
|
|
4956
|
+
arbi,
|
|
4957
|
+
updates
|
|
4958
|
+
);
|
|
4959
|
+
processed += batch.length;
|
|
4960
|
+
console.log(` [${processed}/${docIds.length}] Triggered reprocessing...`);
|
|
4961
|
+
}
|
|
4962
|
+
success(`Triggered reprocessing for ${docIds.length} document(s).`);
|
|
4963
|
+
})()
|
|
4964
|
+
);
|
|
4965
|
+
}
|
|
4966
|
+
function printSummary(summary) {
|
|
4967
|
+
console.log("");
|
|
4968
|
+
console.log("Upload summary:");
|
|
4969
|
+
console.log(` Files found: ${summary.totalFiles}`);
|
|
4970
|
+
success(` Uploaded: ${summary.uploaded}`);
|
|
4971
|
+
if (summary.skipped > 0) {
|
|
4972
|
+
warn(` Skipped: ${summary.skipped}`);
|
|
4973
|
+
}
|
|
4974
|
+
console.log(` Folders: ${summary.folders}`);
|
|
4975
|
+
if (summary.skippedDetails.length > 0) {
|
|
4976
|
+
console.log("");
|
|
4977
|
+
warn("Skipped files:");
|
|
4978
|
+
for (const s of summary.skippedDetails) {
|
|
4979
|
+
warn(` ${s.file_name}: ${s.reason}`);
|
|
4980
|
+
}
|
|
4981
|
+
}
|
|
4758
4982
|
}
|
|
4759
4983
|
function registerUploadCommand(program2) {
|
|
4760
|
-
program2.command("add
|
|
4984
|
+
program2.command("add [paths...]").alias("upload").description("Add files, directories, or zip archives to the active workspace").option("-w, --workspace <id>", "Workspace ID (defaults to selected workspace)").option("-W, --watch", "Watch document processing progress after upload").option("--no-watch", "Skip watching document processing").option("--dry-run", "Show what would be uploaded without uploading").option("--json", "Output results as JSON").option("--config <id>", "Config ext_id to use (e.g. cfg-xxx for SKIP_DUPLICATES)").option("--manifest <file>", "Read newline-delimited paths from file (eDiscovery mode)").option("--log <file>", "Append per-file JSONL audit log (eDiscovery mode)").option("--resume", "Skip paths already completed in --log (requires --log)").option("--fail-fast", "Abort on first upload error").option("--root <dir>", "Root directory for backend folder paths (defaults to common parent)").option(
|
|
4985
|
+
"--limit <n>",
|
|
4986
|
+
"Max files to upload this run (applied after --resume filter, manifest mode only)"
|
|
4987
|
+
).action(
|
|
4761
4988
|
(paths, opts) => runAction(async () => {
|
|
4762
|
-
|
|
4763
|
-
|
|
4989
|
+
const isManifestMode = Boolean(opts.manifest || opts.log || opts.resume);
|
|
4990
|
+
const inputPaths = paths ?? [];
|
|
4991
|
+
if (isManifestMode) {
|
|
4992
|
+
await runManifestMode(inputPaths, opts);
|
|
4993
|
+
return;
|
|
4994
|
+
}
|
|
4995
|
+
if (inputPaths.length === 0) {
|
|
4996
|
+
error("No paths provided. Pass file/dir args or use --manifest <file>.");
|
|
4997
|
+
process.exit(1);
|
|
4998
|
+
}
|
|
4999
|
+
for (const p of inputPaths) {
|
|
5000
|
+
if (!fs4__default.default.existsSync(p)) {
|
|
4764
5001
|
error(`Path not found: ${p}`);
|
|
4765
5002
|
process.exit(1);
|
|
4766
5003
|
}
|
|
4767
5004
|
}
|
|
4768
|
-
const
|
|
5005
|
+
const isInteractive = process.stdout.isTTY === true;
|
|
5006
|
+
const watchPref = getConfig()?.watch !== false;
|
|
5007
|
+
const willWatch = opts.watch === false ? false : opts.watch === true || watchPref && isInteractive;
|
|
5008
|
+
const { config, accessToken, workspaceId } = await resolveWorkspace(opts.workspace, {
|
|
5009
|
+
skipNotifications: willWatch
|
|
5010
|
+
});
|
|
4769
5011
|
const uploadedDocs = /* @__PURE__ */ new Map();
|
|
4770
5012
|
const auth = { baseUrl: config.baseUrl, accessToken };
|
|
4771
|
-
|
|
4772
|
-
|
|
5013
|
+
const summary = {
|
|
5014
|
+
totalFiles: 0,
|
|
5015
|
+
uploaded: 0,
|
|
5016
|
+
skipped: 0,
|
|
5017
|
+
duplicates: 0,
|
|
5018
|
+
folders: 0,
|
|
5019
|
+
docIds: [],
|
|
5020
|
+
skippedDetails: []
|
|
5021
|
+
};
|
|
5022
|
+
const progressCallbacks = {
|
|
5023
|
+
configExtId: opts.config,
|
|
5024
|
+
onBatchStart: (progress) => {
|
|
5025
|
+
if (!opts.json) {
|
|
5026
|
+
dim(
|
|
5027
|
+
` [${progress.batch}/${progress.totalBatches}] Uploading ${progress.filesInBatch} file(s) from ${progress.folder}...`
|
|
5028
|
+
);
|
|
5029
|
+
}
|
|
5030
|
+
},
|
|
5031
|
+
onBatchComplete: (progress) => {
|
|
5032
|
+
const uploaded = progress.result.doc_ext_ids?.length ?? 0;
|
|
5033
|
+
const skipped = progress.result.skipped?.length ?? 0;
|
|
5034
|
+
if (!opts.json) {
|
|
5035
|
+
success(
|
|
5036
|
+
` [${progress.batch}/${progress.totalBatches}] ${uploaded} uploaded, ${skipped} skipped (${progress.filesUploaded}/${progress.totalFiles} total)`
|
|
5037
|
+
);
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
};
|
|
5041
|
+
for (const filePath of inputPaths) {
|
|
5042
|
+
const stat = fs4__default.default.statSync(filePath);
|
|
4773
5043
|
if (stat.isDirectory()) {
|
|
4774
|
-
|
|
4775
|
-
|
|
5044
|
+
if (opts.dryRun) {
|
|
5045
|
+
const count = countSupportedFiles(filePath);
|
|
5046
|
+
console.log(`${filePath}: ${count} supported file(s) would be uploaded`);
|
|
5047
|
+
summary.totalFiles += count;
|
|
5048
|
+
continue;
|
|
5049
|
+
}
|
|
5050
|
+
const result = await sdk.documentsNode.uploadDirectory(
|
|
5051
|
+
auth,
|
|
5052
|
+
workspaceId,
|
|
5053
|
+
filePath,
|
|
5054
|
+
progressCallbacks
|
|
5055
|
+
);
|
|
5056
|
+
if (result.doc_ext_ids.length === 0 && result.skipped.length === 0) {
|
|
4776
5057
|
warn(`No supported files found in directory: ${filePath}`);
|
|
4777
5058
|
continue;
|
|
4778
5059
|
}
|
|
4779
|
-
|
|
5060
|
+
if (!opts.json) {
|
|
5061
|
+
for (const [folder, info] of result.folders) {
|
|
5062
|
+
success(
|
|
5063
|
+
` ${folder}: ${info.fileCount} file(s) \u2192 ${info.doc_ext_ids.length} uploaded`
|
|
5064
|
+
);
|
|
5065
|
+
if (info.skipped.length > 0) {
|
|
5066
|
+
warn(
|
|
5067
|
+
` Skipped: ${info.skipped.map((s) => `${s.file_name} (${s.reason})`).join(", ")}`
|
|
5068
|
+
);
|
|
5069
|
+
}
|
|
5070
|
+
}
|
|
4780
5071
|
success(
|
|
4781
|
-
`
|
|
5072
|
+
`Uploaded directory: ${filePath} (${result.doc_ext_ids.length} document(s) total)`
|
|
4782
5073
|
);
|
|
4783
|
-
if (info.duplicates.length > 0) {
|
|
4784
|
-
warn(` Duplicates: ${info.duplicates.join(", ")}`);
|
|
4785
|
-
}
|
|
4786
5074
|
}
|
|
4787
|
-
|
|
4788
|
-
`Uploaded directory: ${filePath} (${result.doc_ext_ids.length} document(s) total)`
|
|
4789
|
-
);
|
|
5075
|
+
updateSummary(summary, result);
|
|
4790
5076
|
for (const id of result.doc_ext_ids) uploadedDocs.set(id, filePath);
|
|
4791
5077
|
} else if (filePath.toLowerCase().endsWith(".zip")) {
|
|
4792
|
-
|
|
4793
|
-
|
|
5078
|
+
if (opts.dryRun) {
|
|
5079
|
+
console.log(`${filePath}: zip archive (contents would be extracted and uploaded)`);
|
|
5080
|
+
continue;
|
|
5081
|
+
}
|
|
5082
|
+
const result = await sdk.documentsNode.uploadZip(
|
|
5083
|
+
auth,
|
|
5084
|
+
workspaceId,
|
|
5085
|
+
filePath,
|
|
5086
|
+
progressCallbacks
|
|
5087
|
+
);
|
|
5088
|
+
if (result.doc_ext_ids.length === 0 && result.skipped.length === 0) {
|
|
4794
5089
|
warn(`No supported files found in zip: ${filePath}`);
|
|
4795
5090
|
continue;
|
|
4796
5091
|
}
|
|
4797
|
-
|
|
5092
|
+
if (!opts.json) {
|
|
5093
|
+
for (const [folder, info] of result.folders) {
|
|
5094
|
+
success(
|
|
5095
|
+
` ${folder}: ${info.fileCount} file(s) \u2192 ${info.doc_ext_ids.length} uploaded`
|
|
5096
|
+
);
|
|
5097
|
+
if (info.skipped.length > 0) {
|
|
5098
|
+
warn(
|
|
5099
|
+
` Skipped: ${info.skipped.map((s) => `${s.file_name} (${s.reason})`).join(", ")}`
|
|
5100
|
+
);
|
|
5101
|
+
}
|
|
5102
|
+
}
|
|
5103
|
+
success(`Uploaded zip: ${filePath} (${result.doc_ext_ids.length} document(s) total)`);
|
|
5104
|
+
}
|
|
5105
|
+
updateSummary(summary, result);
|
|
5106
|
+
for (const id of result.doc_ext_ids) uploadedDocs.set(id, filePath);
|
|
5107
|
+
} else if (sdk.documentsNode.ARCHIVE_EXTENSIONS.has(
|
|
5108
|
+
filePath.slice(filePath.lastIndexOf(".")).toLowerCase()
|
|
5109
|
+
)) {
|
|
5110
|
+
if (opts.dryRun) {
|
|
5111
|
+
console.log(`${filePath}: archive (contents would be extracted and uploaded)`);
|
|
5112
|
+
continue;
|
|
5113
|
+
}
|
|
5114
|
+
const result = await sdk.documentsNode.uploadArchive(
|
|
5115
|
+
auth,
|
|
5116
|
+
workspaceId,
|
|
5117
|
+
filePath,
|
|
5118
|
+
progressCallbacks
|
|
5119
|
+
);
|
|
5120
|
+
if (result.doc_ext_ids.length === 0 && result.skipped.length === 0) {
|
|
5121
|
+
warn(`No supported files found in archive: ${filePath}`);
|
|
5122
|
+
continue;
|
|
5123
|
+
}
|
|
5124
|
+
if (!opts.json) {
|
|
5125
|
+
for (const [folder, info] of result.folders) {
|
|
5126
|
+
success(
|
|
5127
|
+
` ${folder}: ${info.fileCount} file(s) \u2192 ${info.doc_ext_ids.length} uploaded`
|
|
5128
|
+
);
|
|
5129
|
+
if (info.skipped.length > 0) {
|
|
5130
|
+
warn(
|
|
5131
|
+
` Skipped: ${info.skipped.map((s) => `${s.file_name} (${s.reason})`).join(", ")}`
|
|
5132
|
+
);
|
|
5133
|
+
}
|
|
5134
|
+
}
|
|
4798
5135
|
success(
|
|
4799
|
-
`
|
|
5136
|
+
`Uploaded archive: ${filePath} (${result.doc_ext_ids.length} document(s) total)`
|
|
4800
5137
|
);
|
|
4801
|
-
if (info.duplicates.length > 0) {
|
|
4802
|
-
warn(` Duplicates: ${info.duplicates.join(", ")}`);
|
|
4803
|
-
}
|
|
4804
5138
|
}
|
|
4805
|
-
|
|
5139
|
+
updateSummary(summary, result);
|
|
4806
5140
|
for (const id of result.doc_ext_ids) uploadedDocs.set(id, filePath);
|
|
4807
5141
|
} else {
|
|
5142
|
+
if (opts.dryRun) {
|
|
5143
|
+
console.log(`${filePath}: single file would be uploaded`);
|
|
5144
|
+
summary.totalFiles += 1;
|
|
5145
|
+
continue;
|
|
5146
|
+
}
|
|
4808
5147
|
const result = await sdk.documentsNode.uploadLocalFile(auth, workspaceId, filePath);
|
|
4809
|
-
|
|
4810
|
-
|
|
4811
|
-
|
|
5148
|
+
if (!opts.json) {
|
|
5149
|
+
success(`Uploaded: ${result.fileName} (${(result.doc_ext_ids ?? []).join(", ")})`);
|
|
5150
|
+
if (result.skipped && result.skipped.length > 0) {
|
|
5151
|
+
warn(
|
|
5152
|
+
` Skipped: ${result.skipped.map((s) => `${s.file_name} (${s.reason})`).join(", ")}`
|
|
5153
|
+
);
|
|
5154
|
+
}
|
|
5155
|
+
}
|
|
5156
|
+
summary.totalFiles += 1;
|
|
5157
|
+
summary.uploaded += result.doc_ext_ids?.length ?? 0;
|
|
5158
|
+
summary.docIds.push(...result.doc_ext_ids ?? []);
|
|
5159
|
+
if (result.skipped) {
|
|
5160
|
+
summary.skipped += result.skipped.length;
|
|
5161
|
+
summary.skippedDetails.push(...result.skipped);
|
|
4812
5162
|
}
|
|
4813
|
-
for (const id of result.doc_ext_ids) uploadedDocs.set(id, result.fileName);
|
|
5163
|
+
for (const id of result.doc_ext_ids ?? []) uploadedDocs.set(id, result.fileName);
|
|
4814
5164
|
}
|
|
4815
5165
|
}
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
if (
|
|
5166
|
+
if (opts.dryRun) {
|
|
5167
|
+
return;
|
|
5168
|
+
}
|
|
5169
|
+
if (opts.json) {
|
|
5170
|
+
console.log(JSON.stringify(summary));
|
|
5171
|
+
} else {
|
|
5172
|
+
printSummary(summary);
|
|
5173
|
+
}
|
|
5174
|
+
if (willWatch && uploadedDocs.size > 0) {
|
|
4820
5175
|
const pending = new Set(uploadedDocs.keys());
|
|
4821
5176
|
const failed = /* @__PURE__ */ new Map();
|
|
4822
5177
|
console.log(`
|
|
@@ -4868,7 +5223,7 @@ Connection closed. ${pending.size} document(s) still processing.`);
|
|
|
4868
5223
|
}
|
|
4869
5224
|
});
|
|
4870
5225
|
await done;
|
|
4871
|
-
} else if (uploadedDocs.size > 0 && !
|
|
5226
|
+
} else if (uploadedDocs.size > 0 && !willWatch) {
|
|
4872
5227
|
dim(
|
|
4873
5228
|
'Tip: Use -W/--watch to monitor processing progress, or run "arbi docs" to check status.'
|
|
4874
5229
|
);
|
|
@@ -4876,6 +5231,501 @@ Connection closed. ${pending.size} document(s) still processing.`);
|
|
|
4876
5231
|
})()
|
|
4877
5232
|
);
|
|
4878
5233
|
}
|
|
5234
|
+
function updateSummary(summary, result) {
|
|
5235
|
+
let totalFiles = 0;
|
|
5236
|
+
for (const [, info] of result.folders) {
|
|
5237
|
+
totalFiles += info.fileCount;
|
|
5238
|
+
summary.folders += 1;
|
|
5239
|
+
}
|
|
5240
|
+
summary.totalFiles += totalFiles;
|
|
5241
|
+
summary.uploaded += result.doc_ext_ids.length;
|
|
5242
|
+
summary.skipped += result.skipped.length;
|
|
5243
|
+
summary.docIds.push(...result.doc_ext_ids);
|
|
5244
|
+
summary.skippedDetails.push(...result.skipped);
|
|
5245
|
+
}
|
|
5246
|
+
function countSupportedFiles(dirPath) {
|
|
5247
|
+
let count = 0;
|
|
5248
|
+
for (const entry of fs4__default.default.readdirSync(dirPath, { withFileTypes: true })) {
|
|
5249
|
+
const fullPath = `${dirPath}/${entry.name}`;
|
|
5250
|
+
if (entry.isDirectory()) {
|
|
5251
|
+
count += countSupportedFiles(fullPath);
|
|
5252
|
+
} else if (entry.isFile()) {
|
|
5253
|
+
const ext = entry.name.toLowerCase().lastIndexOf(".");
|
|
5254
|
+
if (ext >= 0) {
|
|
5255
|
+
const extStr = entry.name.slice(ext).toLowerCase();
|
|
5256
|
+
if (sdk.documents.SUPPORTED_EXTENSIONS.has(extStr)) {
|
|
5257
|
+
count++;
|
|
5258
|
+
}
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
}
|
|
5262
|
+
return count;
|
|
5263
|
+
}
|
|
5264
|
+
function readLogState(logPath) {
|
|
5265
|
+
const state = /* @__PURE__ */ new Map();
|
|
5266
|
+
if (!fs4__default.default.existsSync(logPath)) return state;
|
|
5267
|
+
const text = fs4__default.default.readFileSync(logPath, "utf8");
|
|
5268
|
+
for (const line of text.split("\n")) {
|
|
5269
|
+
const trimmed = line.trim();
|
|
5270
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
5271
|
+
try {
|
|
5272
|
+
const rec = JSON.parse(trimmed);
|
|
5273
|
+
if (rec.event) continue;
|
|
5274
|
+
if (typeof rec.path === "string" && typeof rec.status === "string") {
|
|
5275
|
+
state.set(rec.path, rec.status);
|
|
5276
|
+
}
|
|
5277
|
+
} catch {
|
|
5278
|
+
}
|
|
5279
|
+
}
|
|
5280
|
+
return state;
|
|
5281
|
+
}
|
|
5282
|
+
function readTextManifest(manifestPath) {
|
|
5283
|
+
const text = fs4__default.default.readFileSync(manifestPath, "utf8");
|
|
5284
|
+
const out = [];
|
|
5285
|
+
for (const raw of text.split("\n")) {
|
|
5286
|
+
const line = raw.replace(/\r$/, "").trim();
|
|
5287
|
+
if (!line || line.startsWith("#")) continue;
|
|
5288
|
+
out.push(line);
|
|
5289
|
+
}
|
|
5290
|
+
return out;
|
|
5291
|
+
}
|
|
5292
|
+
function parseCsv(text) {
|
|
5293
|
+
const rows = [];
|
|
5294
|
+
let row = [];
|
|
5295
|
+
let cur = "";
|
|
5296
|
+
let inQuotes = false;
|
|
5297
|
+
let i = 0;
|
|
5298
|
+
while (i < text.length) {
|
|
5299
|
+
const ch = text[i];
|
|
5300
|
+
if (inQuotes) {
|
|
5301
|
+
if (ch === '"') {
|
|
5302
|
+
if (text[i + 1] === '"') {
|
|
5303
|
+
cur += '"';
|
|
5304
|
+
i += 2;
|
|
5305
|
+
continue;
|
|
5306
|
+
}
|
|
5307
|
+
inQuotes = false;
|
|
5308
|
+
i++;
|
|
5309
|
+
} else {
|
|
5310
|
+
cur += ch;
|
|
5311
|
+
i++;
|
|
5312
|
+
}
|
|
5313
|
+
} else if (ch === '"' && cur === "") {
|
|
5314
|
+
inQuotes = true;
|
|
5315
|
+
i++;
|
|
5316
|
+
} else if (ch === ",") {
|
|
5317
|
+
row.push(cur);
|
|
5318
|
+
cur = "";
|
|
5319
|
+
i++;
|
|
5320
|
+
} else if (ch === "\n" || ch === "\r") {
|
|
5321
|
+
row.push(cur);
|
|
5322
|
+
cur = "";
|
|
5323
|
+
rows.push(row);
|
|
5324
|
+
row = [];
|
|
5325
|
+
if (ch === "\r" && text[i + 1] === "\n") i += 2;
|
|
5326
|
+
else i++;
|
|
5327
|
+
} else {
|
|
5328
|
+
cur += ch;
|
|
5329
|
+
i++;
|
|
5330
|
+
}
|
|
5331
|
+
}
|
|
5332
|
+
if (cur !== "" || row.length > 0) {
|
|
5333
|
+
row.push(cur);
|
|
5334
|
+
rows.push(row);
|
|
5335
|
+
}
|
|
5336
|
+
if (rows.length > 0 && rows[rows.length - 1].length === 1 && rows[rows.length - 1][0] === "") {
|
|
5337
|
+
rows.pop();
|
|
5338
|
+
}
|
|
5339
|
+
return rows;
|
|
5340
|
+
}
|
|
5341
|
+
function csvField(v) {
|
|
5342
|
+
if (v.includes(",") || v.includes('"') || v.includes("\n") || v.includes("\r")) {
|
|
5343
|
+
return '"' + v.replace(/"/g, '""') + '"';
|
|
5344
|
+
}
|
|
5345
|
+
return v;
|
|
5346
|
+
}
|
|
5347
|
+
function writeCsvAtomic(filePath, header, rows) {
|
|
5348
|
+
const lines = [header.map(csvField).join(",")];
|
|
5349
|
+
for (const row of rows) {
|
|
5350
|
+
const padded = row.slice();
|
|
5351
|
+
while (padded.length < header.length) padded.push("");
|
|
5352
|
+
lines.push(padded.map(csvField).join(","));
|
|
5353
|
+
}
|
|
5354
|
+
const tmp = filePath + ".tmp";
|
|
5355
|
+
fs4__default.default.writeFileSync(tmp, lines.join("\n") + "\n");
|
|
5356
|
+
fs4__default.default.renameSync(tmp, filePath);
|
|
5357
|
+
}
|
|
5358
|
+
function findColumn(header, candidates) {
|
|
5359
|
+
const normalized = header.map((h) => h.trim().toLowerCase());
|
|
5360
|
+
for (const cand of candidates) {
|
|
5361
|
+
const i = normalized.indexOf(cand.toLowerCase());
|
|
5362
|
+
if (i >= 0) return i;
|
|
5363
|
+
}
|
|
5364
|
+
return -1;
|
|
5365
|
+
}
|
|
5366
|
+
function loadCsvManifest(csvPath, rootDir) {
|
|
5367
|
+
const text = fs4__default.default.readFileSync(csvPath, "utf8");
|
|
5368
|
+
const all = parseCsv(text);
|
|
5369
|
+
if (all.length === 0) {
|
|
5370
|
+
error(`Empty CSV: ${csvPath}`);
|
|
5371
|
+
process.exit(1);
|
|
5372
|
+
}
|
|
5373
|
+
const header = all[0];
|
|
5374
|
+
const rows = all.slice(1);
|
|
5375
|
+
const nameCol = findColumn(header, ["Document Name", "Name", "File Name", "Filename"]);
|
|
5376
|
+
if (nameCol < 0) {
|
|
5377
|
+
error(`CSV ${csvPath} is missing a Document Name / Name column.`);
|
|
5378
|
+
process.exit(1);
|
|
5379
|
+
}
|
|
5380
|
+
const folderCol = findColumn(header, ["Folder", "Path", "Directory"]);
|
|
5381
|
+
if (folderCol < 0) {
|
|
5382
|
+
error(`CSV ${csvPath} is missing a Folder / Path column.`);
|
|
5383
|
+
process.exit(1);
|
|
5384
|
+
}
|
|
5385
|
+
let docIdCol = findColumn(header, ["ARBI Doc ID", "Doc ID", "DocId"]);
|
|
5386
|
+
let statusCol = findColumn(header, ["ARBI Status", "Status"]);
|
|
5387
|
+
if (docIdCol < 0) {
|
|
5388
|
+
header.push("ARBI Doc ID");
|
|
5389
|
+
docIdCol = header.length - 1;
|
|
5390
|
+
}
|
|
5391
|
+
if (statusCol < 0) {
|
|
5392
|
+
header.push("ARBI Status");
|
|
5393
|
+
statusCol = header.length - 1;
|
|
5394
|
+
}
|
|
5395
|
+
for (const row of rows) {
|
|
5396
|
+
while (row.length < header.length) row.push("");
|
|
5397
|
+
}
|
|
5398
|
+
const root = rootDir ? path4__default.default.resolve(rootDir) : process.cwd();
|
|
5399
|
+
const paths = [];
|
|
5400
|
+
const pathToRow = /* @__PURE__ */ new Map();
|
|
5401
|
+
for (let i = 0; i < rows.length; i++) {
|
|
5402
|
+
const row = rows[i];
|
|
5403
|
+
const name = (row[nameCol] ?? "").trim();
|
|
5404
|
+
if (!name) continue;
|
|
5405
|
+
const folder = (row[folderCol] ?? "").trim();
|
|
5406
|
+
const rel = folder ? path4__default.default.join(folder, name) : name;
|
|
5407
|
+
const abs = path4__default.default.resolve(root, rel);
|
|
5408
|
+
paths.push(abs);
|
|
5409
|
+
pathToRow.set(abs, i);
|
|
5410
|
+
}
|
|
5411
|
+
return {
|
|
5412
|
+
filePath: csvPath,
|
|
5413
|
+
header,
|
|
5414
|
+
rows,
|
|
5415
|
+
nameCol,
|
|
5416
|
+
folderCol,
|
|
5417
|
+
docIdCol,
|
|
5418
|
+
statusCol,
|
|
5419
|
+
paths,
|
|
5420
|
+
pathToRow
|
|
5421
|
+
};
|
|
5422
|
+
}
|
|
5423
|
+
async function runManifestMode(inputPaths, opts) {
|
|
5424
|
+
if (opts.resume && !opts.log) {
|
|
5425
|
+
error("--resume requires --log <file>");
|
|
5426
|
+
process.exit(1);
|
|
5427
|
+
}
|
|
5428
|
+
const rawPaths = [];
|
|
5429
|
+
let csv = null;
|
|
5430
|
+
if (opts.manifest) {
|
|
5431
|
+
if (!fs4__default.default.existsSync(opts.manifest)) {
|
|
5432
|
+
error(`Manifest not found: ${opts.manifest}`);
|
|
5433
|
+
process.exit(1);
|
|
5434
|
+
}
|
|
5435
|
+
const ext = path4__default.default.extname(opts.manifest).toLowerCase();
|
|
5436
|
+
if (ext === ".csv" || ext === ".tsv") {
|
|
5437
|
+
csv = loadCsvManifest(opts.manifest, opts.root);
|
|
5438
|
+
rawPaths.push(...csv.paths);
|
|
5439
|
+
} else {
|
|
5440
|
+
rawPaths.push(...readTextManifest(opts.manifest));
|
|
5441
|
+
}
|
|
5442
|
+
}
|
|
5443
|
+
for (const p of inputPaths) {
|
|
5444
|
+
if (!fs4__default.default.existsSync(p)) {
|
|
5445
|
+
error(`Path not found: ${p}`);
|
|
5446
|
+
process.exit(1);
|
|
5447
|
+
}
|
|
5448
|
+
const stat = fs4__default.default.statSync(p);
|
|
5449
|
+
if (stat.isDirectory()) {
|
|
5450
|
+
rawPaths.push(...sdk.documentsNode.walkSupportedFiles(p));
|
|
5451
|
+
} else {
|
|
5452
|
+
rawPaths.push(path4__default.default.resolve(p));
|
|
5453
|
+
}
|
|
5454
|
+
}
|
|
5455
|
+
if (rawPaths.length === 0) {
|
|
5456
|
+
error("Manifest is empty. Nothing to upload.");
|
|
5457
|
+
process.exit(1);
|
|
5458
|
+
}
|
|
5459
|
+
let pathsToUpload = rawPaths;
|
|
5460
|
+
const priorState = opts.log ? readLogState(opts.log) : /* @__PURE__ */ new Map();
|
|
5461
|
+
let resumedCount = 0;
|
|
5462
|
+
if (opts.resume) {
|
|
5463
|
+
pathsToUpload = rawPaths.filter((p) => {
|
|
5464
|
+
const prior = priorState.get(p);
|
|
5465
|
+
if (prior === "uploaded" || prior === "duplicate" || prior === "rejected" || prior === "skipped") {
|
|
5466
|
+
resumedCount++;
|
|
5467
|
+
return false;
|
|
5468
|
+
}
|
|
5469
|
+
return true;
|
|
5470
|
+
});
|
|
5471
|
+
}
|
|
5472
|
+
let limit = null;
|
|
5473
|
+
let limitedOut = 0;
|
|
5474
|
+
if (opts.limit !== void 0) {
|
|
5475
|
+
const n = Number.parseInt(opts.limit, 10);
|
|
5476
|
+
if (!Number.isFinite(n) || n <= 0) {
|
|
5477
|
+
error(`Invalid --limit: ${opts.limit}. Must be a positive integer.`);
|
|
5478
|
+
process.exit(1);
|
|
5479
|
+
}
|
|
5480
|
+
limit = n;
|
|
5481
|
+
if (pathsToUpload.length > n) {
|
|
5482
|
+
limitedOut = pathsToUpload.length - n;
|
|
5483
|
+
pathsToUpload = pathsToUpload.slice(0, n);
|
|
5484
|
+
}
|
|
5485
|
+
}
|
|
5486
|
+
const isInteractive = process.stdout.isTTY === true;
|
|
5487
|
+
const humanOutput = !opts.json;
|
|
5488
|
+
if (humanOutput) {
|
|
5489
|
+
console.log(
|
|
5490
|
+
`Manifest: ${rawPaths.length} file(s)` + (csv ? ` from ${path4__default.default.basename(csv.filePath)}` : "") + (resumedCount ? ` (${resumedCount} already done, skipping)` : "") + (limit !== null && limitedOut > 0 ? ` (limit=${limit}, ${limitedOut} deferred to next run)` : "")
|
|
5491
|
+
);
|
|
5492
|
+
}
|
|
5493
|
+
if (opts.dryRun) {
|
|
5494
|
+
if (opts.json) {
|
|
5495
|
+
console.log(
|
|
5496
|
+
JSON.stringify({
|
|
5497
|
+
total: rawPaths.length,
|
|
5498
|
+
resumed: resumedCount,
|
|
5499
|
+
toUpload: pathsToUpload.length
|
|
5500
|
+
})
|
|
5501
|
+
);
|
|
5502
|
+
} else {
|
|
5503
|
+
console.log(`Would upload ${pathsToUpload.length} file(s)`);
|
|
5504
|
+
}
|
|
5505
|
+
return;
|
|
5506
|
+
}
|
|
5507
|
+
if (pathsToUpload.length === 0) {
|
|
5508
|
+
if (humanOutput) success("Nothing to do \u2014 all paths already completed in log.");
|
|
5509
|
+
return;
|
|
5510
|
+
}
|
|
5511
|
+
const watchPref = getConfig()?.watch !== false;
|
|
5512
|
+
const willWatch = opts.watch === false ? false : opts.watch === true || watchPref && isInteractive;
|
|
5513
|
+
const { config, accessToken, workspaceId } = await resolveWorkspace(opts.workspace, {
|
|
5514
|
+
skipNotifications: willWatch
|
|
5515
|
+
});
|
|
5516
|
+
const auth = { baseUrl: config.baseUrl, accessToken };
|
|
5517
|
+
let logFd = null;
|
|
5518
|
+
if (opts.log) {
|
|
5519
|
+
const dir = path4__default.default.dirname(path4__default.default.resolve(opts.log));
|
|
5520
|
+
fs4__default.default.mkdirSync(dir, { recursive: true });
|
|
5521
|
+
logFd = fs4__default.default.openSync(opts.log, "a");
|
|
5522
|
+
const header = JSON.stringify({
|
|
5523
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5524
|
+
event: "run_start",
|
|
5525
|
+
workspace: workspaceId,
|
|
5526
|
+
config: opts.config ?? null,
|
|
5527
|
+
inputs: rawPaths.length,
|
|
5528
|
+
toUpload: pathsToUpload.length,
|
|
5529
|
+
resumed: resumedCount
|
|
5530
|
+
});
|
|
5531
|
+
fs4__default.default.writeSync(logFd, header + "\n");
|
|
5532
|
+
fs4__default.default.fsyncSync(logFd);
|
|
5533
|
+
}
|
|
5534
|
+
const uploadedDocs = /* @__PURE__ */ new Map();
|
|
5535
|
+
const runStart = Date.now();
|
|
5536
|
+
const writeLog = (o) => {
|
|
5537
|
+
if (logFd === null) return;
|
|
5538
|
+
const rec = {
|
|
5539
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5540
|
+
path: o.path,
|
|
5541
|
+
folder: o.folder,
|
|
5542
|
+
sizeBytes: o.sizeBytes,
|
|
5543
|
+
status: o.status,
|
|
5544
|
+
...o.docId ? { docId: o.docId } : {},
|
|
5545
|
+
...o.reason ? { reason: o.reason } : {},
|
|
5546
|
+
attempt: o.attempt,
|
|
5547
|
+
...o.batchId ? { batchId: o.batchId } : {}
|
|
5548
|
+
};
|
|
5549
|
+
fs4__default.default.writeSync(logFd, JSON.stringify(rec) + "\n");
|
|
5550
|
+
fs4__default.default.fsyncSync(logFd);
|
|
5551
|
+
};
|
|
5552
|
+
const total = pathsToUpload.length;
|
|
5553
|
+
let done = 0;
|
|
5554
|
+
let uploadedCount = 0;
|
|
5555
|
+
let duplicateCount = 0;
|
|
5556
|
+
let rejectedCount = 0;
|
|
5557
|
+
let errorCount = 0;
|
|
5558
|
+
let skippedCount = 0;
|
|
5559
|
+
const renderProgress = () => {
|
|
5560
|
+
if (!humanOutput || !isInteractive) return;
|
|
5561
|
+
const pct = total > 0 ? Math.floor(done / total * 100) : 100;
|
|
5562
|
+
const line = ` ${done}/${total} (${pct}%) ok=${uploadedCount}` + (duplicateCount > 0 ? ` dup=${duplicateCount}` : "") + (rejectedCount > 0 ? ` rej=${rejectedCount}` : "") + (skippedCount > 0 ? ` skip=${skippedCount}` : "") + (errorCount > 0 ? ` err=${errorCount}` : "");
|
|
5563
|
+
process.stderr.write("\r" + line.padEnd(80, " "));
|
|
5564
|
+
};
|
|
5565
|
+
const clearProgress = () => {
|
|
5566
|
+
if (humanOutput && isInteractive) process.stderr.write("\r" + " ".repeat(80) + "\r");
|
|
5567
|
+
};
|
|
5568
|
+
const printInlineOutcome = (o) => {
|
|
5569
|
+
if (!humanOutput) return;
|
|
5570
|
+
const name = path4__default.default.basename(o.path);
|
|
5571
|
+
if (o.status === "rejected") {
|
|
5572
|
+
clearProgress();
|
|
5573
|
+
warn(` \u2717 ${name}: ${o.reason}`);
|
|
5574
|
+
} else if (o.status === "error") {
|
|
5575
|
+
clearProgress();
|
|
5576
|
+
error(` ! ${name}: ${o.reason}`);
|
|
5577
|
+
}
|
|
5578
|
+
};
|
|
5579
|
+
let sinceCheckpoint = 0;
|
|
5580
|
+
const CHECKPOINT_EVERY = 25;
|
|
5581
|
+
const csvStatusText = (o) => {
|
|
5582
|
+
if (o.status === "uploaded") return "uploaded";
|
|
5583
|
+
if (o.status === "duplicate") return `duplicate: ${o.reason ?? ""}`.trim();
|
|
5584
|
+
if (o.status === "rejected") return `rejected: ${o.reason ?? ""}`.trim();
|
|
5585
|
+
if (o.status === "skipped") return `skipped: ${o.reason ?? ""}`.trim();
|
|
5586
|
+
return `error: ${o.reason ?? ""}`.trim();
|
|
5587
|
+
};
|
|
5588
|
+
const flushCsv = () => {
|
|
5589
|
+
if (csv === null) return;
|
|
5590
|
+
writeCsvAtomic(csv.filePath, csv.header, csv.rows);
|
|
5591
|
+
};
|
|
5592
|
+
const result = await sdk.documentsNode.uploadManifest(auth, workspaceId, pathsToUpload, {
|
|
5593
|
+
rootDir: opts.root,
|
|
5594
|
+
configExtId: opts.config,
|
|
5595
|
+
failFast: opts.failFast,
|
|
5596
|
+
onFile: async (outcome) => {
|
|
5597
|
+
writeLog(outcome);
|
|
5598
|
+
done++;
|
|
5599
|
+
switch (outcome.status) {
|
|
5600
|
+
case "uploaded":
|
|
5601
|
+
uploadedCount++;
|
|
5602
|
+
if (outcome.docId) uploadedDocs.set(outcome.docId, path4__default.default.basename(outcome.path));
|
|
5603
|
+
break;
|
|
5604
|
+
case "duplicate":
|
|
5605
|
+
duplicateCount++;
|
|
5606
|
+
break;
|
|
5607
|
+
case "rejected":
|
|
5608
|
+
rejectedCount++;
|
|
5609
|
+
break;
|
|
5610
|
+
case "error":
|
|
5611
|
+
errorCount++;
|
|
5612
|
+
break;
|
|
5613
|
+
case "skipped":
|
|
5614
|
+
skippedCount++;
|
|
5615
|
+
break;
|
|
5616
|
+
}
|
|
5617
|
+
printInlineOutcome(outcome);
|
|
5618
|
+
renderProgress();
|
|
5619
|
+
if (csv !== null) {
|
|
5620
|
+
const rowIdx = csv.pathToRow.get(outcome.path);
|
|
5621
|
+
if (rowIdx !== void 0) {
|
|
5622
|
+
if (outcome.docId) csv.rows[rowIdx][csv.docIdCol] = outcome.docId;
|
|
5623
|
+
csv.rows[rowIdx][csv.statusCol] = csvStatusText(outcome);
|
|
5624
|
+
sinceCheckpoint++;
|
|
5625
|
+
if (sinceCheckpoint >= CHECKPOINT_EVERY) {
|
|
5626
|
+
flushCsv();
|
|
5627
|
+
sinceCheckpoint = 0;
|
|
5628
|
+
}
|
|
5629
|
+
}
|
|
5630
|
+
}
|
|
5631
|
+
}
|
|
5632
|
+
});
|
|
5633
|
+
if (csv !== null && sinceCheckpoint > 0) flushCsv();
|
|
5634
|
+
if (logFd !== null) {
|
|
5635
|
+
const footer = JSON.stringify({
|
|
5636
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5637
|
+
event: "run_end",
|
|
5638
|
+
...result.summary,
|
|
5639
|
+
elapsedMs: Date.now() - runStart
|
|
5640
|
+
});
|
|
5641
|
+
fs4__default.default.writeSync(logFd, footer + "\n");
|
|
5642
|
+
fs4__default.default.fsyncSync(logFd);
|
|
5643
|
+
fs4__default.default.closeSync(logFd);
|
|
5644
|
+
}
|
|
5645
|
+
clearProgress();
|
|
5646
|
+
if (opts.json) {
|
|
5647
|
+
console.log(
|
|
5648
|
+
JSON.stringify({
|
|
5649
|
+
...result.summary,
|
|
5650
|
+
elapsedMs: Date.now() - runStart,
|
|
5651
|
+
docIds: result.outcomes.filter((o) => o.docId).map((o) => o.docId)
|
|
5652
|
+
})
|
|
5653
|
+
);
|
|
5654
|
+
} else {
|
|
5655
|
+
const elapsed = ((Date.now() - runStart) / 1e3).toFixed(1);
|
|
5656
|
+
console.log("");
|
|
5657
|
+
console.log("Upload summary");
|
|
5658
|
+
console.log("\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500");
|
|
5659
|
+
console.log(` Files: ${result.summary.total}`);
|
|
5660
|
+
if (result.summary.uploaded > 0) success(` Uploaded: ${result.summary.uploaded}`);
|
|
5661
|
+
if (result.summary.duplicate > 0) dim(` Duplicates: ${result.summary.duplicate}`);
|
|
5662
|
+
if (result.summary.rejected > 0) warn(` Rejected: ${result.summary.rejected}`);
|
|
5663
|
+
if (result.summary.skipped > 0) dim(` Skipped: ${result.summary.skipped}`);
|
|
5664
|
+
if (result.summary.error > 0) error(` Errors: ${result.summary.error}`);
|
|
5665
|
+
console.log(` Elapsed: ${elapsed}s`);
|
|
5666
|
+
if (opts.log) dim(` Log: ${opts.log}`);
|
|
5667
|
+
if (csv) dim(` CSV: ${csv.filePath} (updated)`);
|
|
5668
|
+
}
|
|
5669
|
+
if (result.summary.error > 0) {
|
|
5670
|
+
if (!opts.json) {
|
|
5671
|
+
dim("\nSome files errored. Re-run with --resume to retry failed entries.");
|
|
5672
|
+
}
|
|
5673
|
+
process.exit(1);
|
|
5674
|
+
}
|
|
5675
|
+
if (willWatch && uploadedDocs.size > 0) {
|
|
5676
|
+
const pending = new Set(uploadedDocs.keys());
|
|
5677
|
+
const failed = /* @__PURE__ */ new Map();
|
|
5678
|
+
console.log(`
|
|
5679
|
+
Watching ${pending.size} document(s)...`);
|
|
5680
|
+
let onDone;
|
|
5681
|
+
const done2 = new Promise((r) => {
|
|
5682
|
+
onDone = r;
|
|
5683
|
+
});
|
|
5684
|
+
const conn = await sdk.connectWebSocket({
|
|
5685
|
+
baseUrl: config.baseUrl,
|
|
5686
|
+
accessToken,
|
|
5687
|
+
onMessage: (msg) => {
|
|
5688
|
+
if (client.isMessageType(msg, "task_update")) {
|
|
5689
|
+
if (!pending.has(msg.doc_ext_id)) return;
|
|
5690
|
+
const docName = uploadedDocs.get(msg.doc_ext_id) || msg.file_name;
|
|
5691
|
+
const extra = msg;
|
|
5692
|
+
if (msg.status === "failed") {
|
|
5693
|
+
const reason = extra.error_reason || extra.status_details || extra.detail || "Unknown error";
|
|
5694
|
+
failed.set(msg.doc_ext_id, reason);
|
|
5695
|
+
console.log(` ${docName}: ${status(msg.status)} \u2014 ${reason}`);
|
|
5696
|
+
pending.delete(msg.doc_ext_id);
|
|
5697
|
+
} else {
|
|
5698
|
+
console.log(` ${docName}: ${status(msg.status)} (${msg.progress}%)`);
|
|
5699
|
+
if (msg.status === "completed") {
|
|
5700
|
+
pending.delete(msg.doc_ext_id);
|
|
5701
|
+
}
|
|
5702
|
+
}
|
|
5703
|
+
if (pending.size === 0) {
|
|
5704
|
+
conn.close();
|
|
5705
|
+
}
|
|
5706
|
+
}
|
|
5707
|
+
},
|
|
5708
|
+
onClose: () => {
|
|
5709
|
+
if (failed.size > 0) {
|
|
5710
|
+
error(`
|
|
5711
|
+
${failed.size} document(s) failed to process:`);
|
|
5712
|
+
for (const [docId, reason] of failed) {
|
|
5713
|
+
error(` ${uploadedDocs.get(docId) || docId}: ${reason}`);
|
|
5714
|
+
}
|
|
5715
|
+
}
|
|
5716
|
+
if (pending.size > 0) {
|
|
5717
|
+
warn(`
|
|
5718
|
+
Connection closed. ${pending.size} document(s) still processing.`);
|
|
5719
|
+
dim('Run "arbi watch" to continue monitoring, or "arbi docs" to check status.');
|
|
5720
|
+
} else if (failed.size === 0 && uploadedDocs.size > 0) {
|
|
5721
|
+
success("\nAll documents processed successfully.");
|
|
5722
|
+
}
|
|
5723
|
+
onDone();
|
|
5724
|
+
}
|
|
5725
|
+
});
|
|
5726
|
+
await done2;
|
|
5727
|
+
}
|
|
5728
|
+
}
|
|
4879
5729
|
function registerDownloadCommand(program2) {
|
|
4880
5730
|
program2.command("download [doc-id]").description("Download a document (interactive picker if no ID given)").option("-o, --output <path>", "Output file path").option("-w, --workspace <id>", "Workspace ID (defaults to selected workspace)").action(
|
|
4881
5731
|
(docId, opts) => runAction(async () => {
|
|
@@ -4903,11 +5753,11 @@ function registerDownloadCommand(program2) {
|
|
|
4903
5753
|
const match = disposition.match(/filename[*]?=(?:UTF-8''|"?)([^";]+)/i);
|
|
4904
5754
|
if (match) filename = decodeURIComponent(match[1].replace(/"/g, ""));
|
|
4905
5755
|
}
|
|
4906
|
-
const outputPath = opts.output ||
|
|
5756
|
+
const outputPath = opts.output || path4__default.default.join(process.cwd(), filename);
|
|
4907
5757
|
const buffer = Buffer.from(await res.arrayBuffer());
|
|
4908
|
-
|
|
5758
|
+
fs4__default.default.writeFileSync(outputPath, buffer);
|
|
4909
5759
|
success(
|
|
4910
|
-
`Downloaded: ${
|
|
5760
|
+
`Downloaded: ${path4__default.default.basename(outputPath)} (${(buffer.length / (1024 * 1024)).toFixed(1)} MB)`
|
|
4911
5761
|
);
|
|
4912
5762
|
})()
|
|
4913
5763
|
);
|
|
@@ -5237,7 +6087,9 @@ function colorize2(level, text) {
|
|
|
5237
6087
|
function registerWatchCommand(program2) {
|
|
5238
6088
|
program2.command("watch").description("Watch workspace activity in real time").option("-w, --workspace <id>", "Workspace ID (defaults to selected workspace)").option("-t, --timeout <seconds>", "Auto-close after N seconds").option("-n, --count <n>", "Stop after N messages").option("--json", "Output NDJSON (one JSON object per line)").action(
|
|
5239
6089
|
(opts) => runAction(async () => {
|
|
5240
|
-
const { config, accessToken, workspaceId } = await resolveWorkspace(opts.workspace
|
|
6090
|
+
const { config, accessToken, workspaceId } = await resolveWorkspace(opts.workspace, {
|
|
6091
|
+
skipNotifications: true
|
|
6092
|
+
});
|
|
5241
6093
|
const timeoutSec = opts.timeout ? parseInt(opts.timeout, 10) : void 0;
|
|
5242
6094
|
const maxCount = opts.count ? parseInt(opts.count, 10) : void 0;
|
|
5243
6095
|
const jsonMode = opts.json ?? false;
|
|
@@ -6289,12 +7141,20 @@ function registerQuickstartCommand(program2) {
|
|
|
6289
7141
|
const memberWorkspaces = wsList.filter(
|
|
6290
7142
|
(ws) => ws.users?.some((u) => u.user.email === email)
|
|
6291
7143
|
);
|
|
7144
|
+
const userProjects = await sdk.projects.listProjects(arbi);
|
|
7145
|
+
const defaultProjectExtId = userProjects[0]?.external_id;
|
|
7146
|
+
if (!defaultProjectExtId) throw new Error("No projects found for user");
|
|
6292
7147
|
let workspaceId;
|
|
6293
7148
|
let workspaceName;
|
|
6294
7149
|
if (memberWorkspaces.length === 0) {
|
|
6295
7150
|
console.log("Creating your first workspace...");
|
|
6296
7151
|
const encryptedKey = await sdk.generateNewWorkspaceKey(arbi, loginResult.serverSessionKey);
|
|
6297
|
-
const ws = await sdk.workspaces.createWorkspace(
|
|
7152
|
+
const ws = await sdk.workspaces.createWorkspace(
|
|
7153
|
+
arbi,
|
|
7154
|
+
"My Workspace",
|
|
7155
|
+
encryptedKey,
|
|
7156
|
+
defaultProjectExtId
|
|
7157
|
+
);
|
|
6298
7158
|
workspaceId = ws.external_id;
|
|
6299
7159
|
workspaceName = ws.name;
|
|
6300
7160
|
} else {
|
|
@@ -6306,7 +7166,7 @@ function registerQuickstartCommand(program2) {
|
|
|
6306
7166
|
if (selected === "__new__") {
|
|
6307
7167
|
const name = await promptInput("Workspace name", false) || "My Workspace";
|
|
6308
7168
|
const encKey = await sdk.generateNewWorkspaceKey(arbi, loginResult.serverSessionKey);
|
|
6309
|
-
const ws = await sdk.workspaces.createWorkspace(arbi, name, encKey);
|
|
7169
|
+
const ws = await sdk.workspaces.createWorkspace(arbi, name, encKey, defaultProjectExtId);
|
|
6310
7170
|
workspaceId = ws.external_id;
|
|
6311
7171
|
workspaceName = ws.name;
|
|
6312
7172
|
} else {
|
|
@@ -6411,8 +7271,16 @@ function registerAgentCreateCommand(program2) {
|
|
|
6411
7271
|
opts.password,
|
|
6412
7272
|
store
|
|
6413
7273
|
);
|
|
7274
|
+
const userProjects = await sdk.projects.listProjects(arbi);
|
|
7275
|
+
const defaultProjectExtId = userProjects[0]?.external_id;
|
|
7276
|
+
if (!defaultProjectExtId) throw new Error("No projects found for agent");
|
|
6414
7277
|
const encryptedKey = await sdk.generateNewWorkspaceKey(arbi, loginResult.serverSessionKey);
|
|
6415
|
-
const ws = await sdk.workspaces.createWorkspace(
|
|
7278
|
+
const ws = await sdk.workspaces.createWorkspace(
|
|
7279
|
+
arbi,
|
|
7280
|
+
opts.workspaceName,
|
|
7281
|
+
encryptedKey,
|
|
7282
|
+
defaultProjectExtId
|
|
7283
|
+
);
|
|
6416
7284
|
updateConfig({ selectedWorkspaceId: ws.external_id });
|
|
6417
7285
|
console.log("");
|
|
6418
7286
|
success("Agent account created!");
|
|
@@ -6582,7 +7450,7 @@ function resolveTaskId(taskIdArg) {
|
|
|
6582
7450
|
}
|
|
6583
7451
|
return latest.id;
|
|
6584
7452
|
}
|
|
6585
|
-
function
|
|
7453
|
+
function statusColor2(s) {
|
|
6586
7454
|
if (s === "completed") return chalk2__default.default.green(s);
|
|
6587
7455
|
if (s === "failed") return chalk2__default.default.red(s);
|
|
6588
7456
|
if (s === "queued" || s === "in_progress") return chalk2__default.default.yellow(s);
|
|
@@ -6603,7 +7471,7 @@ function registerTaskCommand(program2) {
|
|
|
6603
7471
|
printTable(
|
|
6604
7472
|
[
|
|
6605
7473
|
{ header: "ID", width: 20, value: (r) => r.id },
|
|
6606
|
-
{ header: "STATUS", width: 14, value: (r) =>
|
|
7474
|
+
{ header: "STATUS", width: 14, value: (r) => statusColor2(r.status) },
|
|
6607
7475
|
{ header: "QUESTION", width: 42, value: (r) => r.question },
|
|
6608
7476
|
{ header: "AGE", width: 6, value: (r) => formatAge(r.submittedAt) }
|
|
6609
7477
|
],
|
|
@@ -6623,7 +7491,7 @@ function registerTaskCommand(program2) {
|
|
|
6623
7491
|
console.log(JSON.stringify(result, null, 2));
|
|
6624
7492
|
} else {
|
|
6625
7493
|
console.log(`${chalk2__default.default.bold("ID:")} ${result.id}`);
|
|
6626
|
-
console.log(`${chalk2__default.default.bold("Status:")} ${
|
|
7494
|
+
console.log(`${chalk2__default.default.bold("Status:")} ${statusColor2(result.status)}`);
|
|
6627
7495
|
if (result.model) console.log(`${chalk2__default.default.bold("Model:")} ${result.model}`);
|
|
6628
7496
|
if (result.usage) {
|
|
6629
7497
|
console.log(`${chalk2__default.default.bold("Tokens:")} ${result.usage.total_tokens.toLocaleString()}`);
|
|
@@ -6680,15 +7548,15 @@ compdef _arbi_completions arbi
|
|
|
6680
7548
|
${MARKER_END}`;
|
|
6681
7549
|
function getShellRcPath2() {
|
|
6682
7550
|
const shell = process.env.SHELL || "";
|
|
6683
|
-
if (shell.includes("zsh")) return
|
|
6684
|
-
return
|
|
7551
|
+
if (shell.includes("zsh")) return path4.join(os.homedir(), ".zshrc");
|
|
7552
|
+
return path4.join(os.homedir(), ".bashrc");
|
|
6685
7553
|
}
|
|
6686
7554
|
function isZsh() {
|
|
6687
7555
|
return (process.env.SHELL || "").includes("zsh");
|
|
6688
7556
|
}
|
|
6689
7557
|
function isCompletionInstalled(rcPath) {
|
|
6690
|
-
if (!
|
|
6691
|
-
const content =
|
|
7558
|
+
if (!fs4.existsSync(rcPath)) return false;
|
|
7559
|
+
const content = fs4.readFileSync(rcPath, "utf-8");
|
|
6692
7560
|
return content.includes(MARKER_START);
|
|
6693
7561
|
}
|
|
6694
7562
|
function removeCompletionBlock(content) {
|
|
@@ -6709,7 +7577,7 @@ function registerCompletionCommand(program2) {
|
|
|
6709
7577
|
return;
|
|
6710
7578
|
}
|
|
6711
7579
|
const snippet = isZsh() ? ZSH_COMPLETION : BASH_COMPLETION;
|
|
6712
|
-
|
|
7580
|
+
fs4.appendFileSync(rcPath, snippet + "\n");
|
|
6713
7581
|
success(`Installed tab completion in ${rcPath}`);
|
|
6714
7582
|
dim("");
|
|
6715
7583
|
dim(`Run: source ${rcPath}`);
|
|
@@ -6721,9 +7589,9 @@ function registerCompletionCommand(program2) {
|
|
|
6721
7589
|
dim("Completion is not installed.");
|
|
6722
7590
|
return;
|
|
6723
7591
|
}
|
|
6724
|
-
const content =
|
|
7592
|
+
const content = fs4.readFileSync(rcPath, "utf-8");
|
|
6725
7593
|
const cleaned = removeCompletionBlock(content);
|
|
6726
|
-
|
|
7594
|
+
fs4.writeFileSync(rcPath, cleaned);
|
|
6727
7595
|
success(`Removed tab completion from ${rcPath}`);
|
|
6728
7596
|
dim(`Run: source ${rcPath}`);
|
|
6729
7597
|
});
|
|
@@ -6739,9 +7607,9 @@ function registerCompletionCommand(program2) {
|
|
|
6739
7607
|
function resolvePath(p) {
|
|
6740
7608
|
const raw = p ?? ".";
|
|
6741
7609
|
const expanded = raw.startsWith("~") ? raw.replace("~", os.homedir()) : raw;
|
|
6742
|
-
return
|
|
7610
|
+
return path4.resolve(expanded);
|
|
6743
7611
|
}
|
|
6744
|
-
function
|
|
7612
|
+
function formatSize2(bytes) {
|
|
6745
7613
|
if (bytes < 1024) return `${bytes}B`;
|
|
6746
7614
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}K`;
|
|
6747
7615
|
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)}M`;
|
|
@@ -6749,11 +7617,11 @@ function formatSize(bytes) {
|
|
|
6749
7617
|
}
|
|
6750
7618
|
function registerLocalCommand(program2) {
|
|
6751
7619
|
const local = program2.command("local").description("Local filesystem operations");
|
|
6752
|
-
local.command("ls [path]").description("List files in a directory").option("-l, --long", "Show file sizes and types").action((
|
|
6753
|
-
const dir = resolvePath(
|
|
7620
|
+
local.command("ls [path]").description("List files in a directory").option("-l, --long", "Show file sizes and types").action((path6, opts) => {
|
|
7621
|
+
const dir = resolvePath(path6);
|
|
6754
7622
|
let entries;
|
|
6755
7623
|
try {
|
|
6756
|
-
entries =
|
|
7624
|
+
entries = fs4.readdirSync(dir);
|
|
6757
7625
|
} catch {
|
|
6758
7626
|
console.error(`Cannot read directory: ${dir}`);
|
|
6759
7627
|
process.exit(1);
|
|
@@ -6761,9 +7629,9 @@ function registerLocalCommand(program2) {
|
|
|
6761
7629
|
for (const entry of entries.filter((e) => !e.startsWith(".")).sort()) {
|
|
6762
7630
|
if (opts?.long) {
|
|
6763
7631
|
try {
|
|
6764
|
-
const stat =
|
|
7632
|
+
const stat = fs4.statSync(path4.join(dir, entry));
|
|
6765
7633
|
const type = stat.isDirectory() ? "dir" : "file";
|
|
6766
|
-
const size = stat.isFile() ?
|
|
7634
|
+
const size = stat.isFile() ? formatSize2(stat.size) : "-";
|
|
6767
7635
|
console.log(`${type.padEnd(5)} ${size.padStart(10)} ${entry}`);
|
|
6768
7636
|
} catch {
|
|
6769
7637
|
console.log(`? ${"-".padStart(10)} ${entry}`);
|
|
@@ -6775,7 +7643,7 @@ function registerLocalCommand(program2) {
|
|
|
6775
7643
|
});
|
|
6776
7644
|
local.command("find <pattern>").description('Find files matching a glob (e.g. "**/*.pdf")').option("-d, --dir <path>", "Directory to search in", ".").action((pattern, opts) => {
|
|
6777
7645
|
const dir = resolvePath(opts.dir);
|
|
6778
|
-
const results =
|
|
7646
|
+
const results = fs4.globSync(pattern, { cwd: dir });
|
|
6779
7647
|
if (results.length === 0) {
|
|
6780
7648
|
console.log(`No files matching "${pattern}" in ${dir}`);
|
|
6781
7649
|
return;
|
|
@@ -6788,7 +7656,7 @@ function registerLocalCommand(program2) {
|
|
|
6788
7656
|
const filePath = resolvePath(file);
|
|
6789
7657
|
let content;
|
|
6790
7658
|
try {
|
|
6791
|
-
content =
|
|
7659
|
+
content = fs4.readFileSync(filePath, "utf-8");
|
|
6792
7660
|
} catch {
|
|
6793
7661
|
console.error(`Cannot read file: ${filePath}`);
|
|
6794
7662
|
process.exit(1);
|
|
@@ -6800,10 +7668,10 @@ function registerLocalCommand(program2) {
|
|
|
6800
7668
|
console.log(content);
|
|
6801
7669
|
}
|
|
6802
7670
|
});
|
|
6803
|
-
local.command("tree [path]").description("Show directory tree").option("-d, --depth <n>", "Maximum depth", "3").action((
|
|
6804
|
-
const dir = resolvePath(
|
|
7671
|
+
local.command("tree [path]").description("Show directory tree").option("-d, --depth <n>", "Maximum depth", "3").action((path6, opts) => {
|
|
7672
|
+
const dir = resolvePath(path6);
|
|
6805
7673
|
const maxDepth = parseInt(opts?.depth ?? "3", 10);
|
|
6806
|
-
console.log(
|
|
7674
|
+
console.log(path4.basename(dir) + "/");
|
|
6807
7675
|
printTree(dir, "", maxDepth, 0);
|
|
6808
7676
|
});
|
|
6809
7677
|
}
|
|
@@ -6811,7 +7679,7 @@ function printTree(dir, prefix, maxDepth, depth) {
|
|
|
6811
7679
|
if (depth >= maxDepth) return;
|
|
6812
7680
|
let entries;
|
|
6813
7681
|
try {
|
|
6814
|
-
entries =
|
|
7682
|
+
entries = fs4.readdirSync(dir).filter((e) => !e.startsWith(".")).sort();
|
|
6815
7683
|
} catch {
|
|
6816
7684
|
return;
|
|
6817
7685
|
}
|
|
@@ -6819,10 +7687,10 @@ function printTree(dir, prefix, maxDepth, depth) {
|
|
|
6819
7687
|
const entry = entries[i];
|
|
6820
7688
|
const isLast = i === entries.length - 1;
|
|
6821
7689
|
const connector = isLast ? "\u2514\u2500\u2500 " : "\u251C\u2500\u2500 ";
|
|
6822
|
-
const full =
|
|
7690
|
+
const full = path4.join(dir, entry);
|
|
6823
7691
|
let isDir = false;
|
|
6824
7692
|
try {
|
|
6825
|
-
isDir =
|
|
7693
|
+
isDir = fs4.statSync(full).isDirectory();
|
|
6826
7694
|
} catch {
|
|
6827
7695
|
continue;
|
|
6828
7696
|
}
|
|
@@ -6877,7 +7745,7 @@ console.info = (...args) => {
|
|
|
6877
7745
|
_origInfo(...args);
|
|
6878
7746
|
};
|
|
6879
7747
|
var program = new commander.Command();
|
|
6880
|
-
program.name("arbi").description("ARBI CLI \u2014 interact with ARBI from the terminal").version("0.3.
|
|
7748
|
+
program.name("arbi").description("ARBI CLI \u2014 interact with ARBI from the terminal").version("0.3.30");
|
|
6881
7749
|
registerConfigCommand(program);
|
|
6882
7750
|
registerLoginCommand(program);
|
|
6883
7751
|
registerRegisterCommand(program);
|