@lyra-ai/toolkit 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/QUICKSTART.md +1 -4
- package/README.md +1 -2
- package/bin/toolkit.mjs +10 -21
- package/package.json +1 -1
- package/src/config-cmd.mjs +1 -1
- package/src/flush.mjs +243 -221
- package/src/hook.mjs +35 -21
- package/src/install.mjs +74 -34
- package/src/shared/backlog.mjs +96 -0
- package/src/shared/config.mjs +17 -0
- package/src/shared/http.mjs +29 -0
- package/src/shared/lock.mjs +79 -0
- package/src/shared/settle.mjs +20 -0
- package/src/shared/zip.mjs +89 -0
- package/src/skillhub.mjs +2 -319
- package/src/status.mjs +103 -36
- package/src/uninstall.mjs +26 -13
- package/src/zentao.mjs +57 -2
package/QUICKSTART.md
CHANGED
|
@@ -134,9 +134,6 @@ skillhub list --json
|
|
|
134
134
|
# 诊断问题
|
|
135
135
|
skillhub doctor
|
|
136
136
|
|
|
137
|
-
# 更新技能包
|
|
138
|
-
skillhub update --scope user
|
|
139
|
-
|
|
140
137
|
# 移除
|
|
141
138
|
skillhub remove code-review --scope user
|
|
142
139
|
```
|
|
@@ -150,7 +147,7 @@ skillhub publish ./my-skill
|
|
|
150
147
|
## skillhub 工作流
|
|
151
148
|
|
|
152
149
|
```
|
|
153
|
-
login(认证) → search(发现) → install(安装) → list/doctor(管理)
|
|
150
|
+
login(认证) → search(发现) → install(安装) → list/doctor(管理)
|
|
154
151
|
```
|
|
155
152
|
|
|
156
153
|
- **`login`** 存储 registry token
|
package/README.md
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
安装后,你的 AI 编码数据自动进入组织分析平台,生成专属的多维度效能看板。
|
|
14
14
|
|
|
15
15
|
```bash
|
|
16
|
-
toolkit install --
|
|
16
|
+
toolkit install --registry http://your-platform:38000
|
|
17
17
|
```
|
|
18
18
|
|
|
19
19
|
- 会话结束后自动同步,无需手动操作
|
|
@@ -117,7 +117,6 @@ AI 技能包管理器:搜索、安装、发布、管理 AI coding agent 技能
|
|
|
117
117
|
- `skillhub remove <slug> [--scope user|project] [--agent <profile>] [--json]` → 移除技能包
|
|
118
118
|
- `skillhub doctor [--fix] [--json]` → 诊断技能包安装状态
|
|
119
119
|
- `skillhub publish <path> [--json]` → 发布技能包到 registry
|
|
120
|
-
- `skillhub update [--scope user|project] [--agent <profile>] [--json]` → 更新已安装技能包
|
|
121
120
|
|
|
122
121
|
### Agent profiles(`--agent`)
|
|
123
122
|
|
package/bin/toolkit.mjs
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* toolkit — 研发工具集 CLI 入口
|
|
4
4
|
*
|
|
5
5
|
* 用法:
|
|
6
|
-
* toolkit install
|
|
6
|
+
* toolkit install --registry <url> 安装插件(自动检测 AI Coding Agent)
|
|
7
7
|
* toolkit status 查看状态
|
|
8
8
|
* toolkit flush 手动上传
|
|
9
9
|
* toolkit uninstall [--purge] 卸载
|
|
@@ -23,8 +23,8 @@ const cmd = args[0];
|
|
|
23
23
|
const HELP = `toolkit — 研发工具集
|
|
24
24
|
|
|
25
25
|
命令:
|
|
26
|
-
install --
|
|
27
|
-
|
|
26
|
+
install --registry <url> 安装插件(自动检测本机 AI Coding Agent)
|
|
27
|
+
当前支持: Claude Code (~/.claude)
|
|
28
28
|
status 查看安装状态 + 待同步会话
|
|
29
29
|
flush 手动触发一次同步
|
|
30
30
|
config 配置管理
|
|
@@ -44,29 +44,18 @@ async function main() {
|
|
|
44
44
|
return;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
// toolkit install --
|
|
47
|
+
// toolkit install --registry <url>(自动检测 AI Coding Agent)
|
|
48
48
|
if (cmd === 'install') {
|
|
49
|
-
const agentIdx = args.indexOf('--agent');
|
|
50
|
-
const agent = agentIdx >= 0 ? args[agentIdx + 1] : null;
|
|
51
|
-
if (!agent) {
|
|
52
|
-
console.error('ERROR: 缺少 --agent <name>');
|
|
53
|
-
console.error(' 用法: toolkit install --agent claudecode --registry <url>');
|
|
54
|
-
console.error(' 支持: claudecode');
|
|
55
|
-
process.exit(2);
|
|
56
|
-
}
|
|
57
|
-
if (agent !== 'claudecode') {
|
|
58
|
-
console.error(`ERROR: 暂不支持 agent "${agent}"。目前支持: claudecode`);
|
|
59
|
-
process.exit(2);
|
|
60
|
-
}
|
|
61
49
|
const regIdx = args.indexOf('--registry');
|
|
62
50
|
let registry = regIdx >= 0 ? args[regIdx + 1] : null;
|
|
63
51
|
if (!registry) {
|
|
64
52
|
console.error('ERROR: 缺少 --registry <url>');
|
|
65
|
-
console.error(' 用法: toolkit install --
|
|
53
|
+
console.error(' 用法: toolkit install --registry <url>');
|
|
54
|
+
console.error(' (自动检测本机 AI Coding Agent,当前支持: Claude Code)');
|
|
66
55
|
process.exit(2);
|
|
67
56
|
}
|
|
68
57
|
const { install } = await import(pathToFileURL(path.join(srcDir, 'install.mjs')).href);
|
|
69
|
-
install(registry
|
|
58
|
+
await install(registry);
|
|
70
59
|
return;
|
|
71
60
|
}
|
|
72
61
|
|
|
@@ -77,11 +66,11 @@ async function main() {
|
|
|
77
66
|
return;
|
|
78
67
|
}
|
|
79
68
|
|
|
80
|
-
// toolkit flush
|
|
69
|
+
// toolkit flush → 跑 sweep 模式(排干 backlog)
|
|
81
70
|
if (cmd === 'flush') {
|
|
82
71
|
const flushScript = path.join(srcDir, 'flush.mjs');
|
|
83
|
-
// 同步运行 flush(显示输出)
|
|
84
|
-
const child = spawn(process.execPath, [flushScript], {
|
|
72
|
+
// 同步运行 flush(显示输出);必须传 'sweep' mode,否则新 flush.mjs 报 unknown mode
|
|
73
|
+
const child = spawn(process.execPath, [flushScript, 'sweep'], {
|
|
85
74
|
stdio: 'inherit',
|
|
86
75
|
env: { ...process.env, TOOLKIT_VERBOSE: '1' },
|
|
87
76
|
});
|
package/package.json
CHANGED
package/src/config-cmd.mjs
CHANGED
|
@@ -27,7 +27,7 @@ export function configCmd(args) {
|
|
|
27
27
|
if (!sub || sub === 'list' || sub === 'help' || sub === '--help') {
|
|
28
28
|
const config = load();
|
|
29
29
|
if (!config.registry && !config.uid) {
|
|
30
|
-
console.error('配置文件不存在。先运行: toolkit install
|
|
30
|
+
console.error('配置文件不存在。先运行: toolkit install --registry <url>');
|
|
31
31
|
process.exit(1);
|
|
32
32
|
}
|
|
33
33
|
const pf = config['project-filter'];
|
package/src/flush.mjs
CHANGED
|
@@ -1,19 +1,41 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* flush.mjs —
|
|
4
|
-
*
|
|
5
|
-
*
|
|
3
|
+
* flush.mjs — current / sweep 双模式上报入口。
|
|
4
|
+
*
|
|
5
|
+
* flush current <sid> <filePath> 上传单文件(无锁,无状态写)
|
|
6
|
+
* flush sweep 扫描 backlog(逆游标排干)
|
|
7
|
+
*
|
|
8
|
+
* current 语义: settle-check → 读 → 单文件 zip → POST → 退出。
|
|
9
|
+
* 成功/失败都不写本地状态。若 last_sweep 距今超过 sweep_interval_sec,
|
|
10
|
+
* piggyback 一次 runSweep。
|
|
11
|
+
*
|
|
12
|
+
* sweep 语义: 抢 flush.lock;对每个命中 project-filter 的项目,按 compareKey
|
|
13
|
+
* 降序排干 mtime ≤ installed_at 的 .jsonl 文件(待处理 = compareKey < cursor),
|
|
14
|
+
* 分块 settle→zip→POST;每块后推进游标到块内最老文件 key 并原子写 backlog.json。
|
|
15
|
+
* POST 失败仍推进游标(漏报可接受,避免卡死)。返回 {chunks, files}。
|
|
6
16
|
*/
|
|
7
17
|
import fs from 'node:fs';
|
|
8
|
-
import path from 'node:path';
|
|
9
18
|
import os from 'node:os';
|
|
10
|
-
import
|
|
11
|
-
import { randomUUID } from 'node:crypto';
|
|
19
|
+
import path from 'node:path';
|
|
12
20
|
import { fileURLToPath } from 'node:url';
|
|
13
|
-
import { load as loadConfig,
|
|
21
|
+
import { load as loadConfig, getUploadConfig, LOG_DIR } from './shared/config.mjs';
|
|
22
|
+
import {
|
|
23
|
+
ensureBacklog,
|
|
24
|
+
loadBacklog,
|
|
25
|
+
saveBacklog,
|
|
26
|
+
compareKey,
|
|
27
|
+
topCursor,
|
|
28
|
+
BACKLOG_PATH,
|
|
29
|
+
} from './shared/backlog.mjs';
|
|
30
|
+
import { acquireLock } from './shared/lock.mjs';
|
|
31
|
+
import { createZip } from './shared/zip.mjs';
|
|
32
|
+
import { postZip } from './shared/http.mjs';
|
|
33
|
+
import { isSettled } from './shared/settle.mjs';
|
|
14
34
|
|
|
15
35
|
const HOME = os.homedir();
|
|
16
|
-
const CLAUDE_PROJECTS =
|
|
36
|
+
const CLAUDE_PROJECTS =
|
|
37
|
+
process.env.CLAUDE_PROJECTS_DIR || path.join(HOME, '.claude', 'projects');
|
|
38
|
+
const LOCK_PATH = path.join(path.dirname(BACKLOG_PATH), 'flush.lock');
|
|
17
39
|
|
|
18
40
|
// ── 日志 ──────────────────────────────────────────────
|
|
19
41
|
function log(...args) {
|
|
@@ -26,248 +48,248 @@ function log(...args) {
|
|
|
26
48
|
if (process.env.TOOLKIT_VERBOSE) process.stderr.write(line);
|
|
27
49
|
}
|
|
28
50
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
function loadUploaded() {
|
|
33
|
-
try {
|
|
34
|
-
return JSON.parse(fs.readFileSync(UPLOADED_PATH, 'utf-8'));
|
|
35
|
-
} catch {
|
|
36
|
-
return { uploaded_sids: [], last_flush: null };
|
|
37
|
-
}
|
|
51
|
+
function uploadUrl(config) {
|
|
52
|
+
const base = `${config.registry.replace(/\/$/, '')}${config.contexts?.metrics || '/ai-metrics'}`;
|
|
53
|
+
return `${base}/api/upload`;
|
|
38
54
|
}
|
|
39
55
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
}
|
|
56
|
+
/**
|
|
57
|
+
* current:上传单个文件(无锁,无状态写)。
|
|
58
|
+
* @param {string} sid 会话 id(用于日志)
|
|
59
|
+
* @param {string} filePath JSONL 文件绝对路径
|
|
60
|
+
* @param {{config:object, uploadConfig:object, logFn?:Function}} ctx
|
|
61
|
+
* @returns {Promise<{uploaded:boolean}>}
|
|
62
|
+
*/
|
|
63
|
+
export async function runCurrent(sid, filePath, ctx) {
|
|
64
|
+
const config = ctx.config;
|
|
65
|
+
const u = ctx.uploadConfig;
|
|
66
|
+
const logFn = ctx.logFn || log;
|
|
52
67
|
|
|
53
|
-
//
|
|
54
|
-
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
const projPath = path.join(CLAUDE_PROJECTS, projDir);
|
|
64
|
-
let st;
|
|
65
|
-
try { st = fs.statSync(projPath); } catch { continue; }
|
|
66
|
-
if (!st.isDirectory()) continue;
|
|
67
|
-
for (const file of fs.readdirSync(projPath)) {
|
|
68
|
-
if (!file.endsWith('.jsonl')) continue;
|
|
69
|
-
const sid = file.slice(0, -6); // 去掉 .jsonl
|
|
70
|
-
if (uploadedSet.has(sid)) continue;
|
|
71
|
-
const fp = path.join(projPath, file);
|
|
72
|
-
// 跳过空文件(<100 字节,可能是刚创建的)
|
|
73
|
-
try { if (fs.statSync(fp).size < 100) continue; } catch { continue; }
|
|
74
|
-
found.push({ sid, path: fp, project: projDir, name: file });
|
|
75
|
-
}
|
|
68
|
+
// project-filter 对 current 同样生效:刚结束的会话若不在过滤范围内,跳过。
|
|
69
|
+
// (与 sweep 一致 —— filter 控制「哪些 project 上报」,两条路都不能绕过。)
|
|
70
|
+
const project = path.basename(path.dirname(filePath));
|
|
71
|
+
const keywords = config['project-filter'] || [];
|
|
72
|
+
if (
|
|
73
|
+
keywords.length &&
|
|
74
|
+
!keywords.some((kw) => project.toLowerCase().includes(kw.toLowerCase()))
|
|
75
|
+
) {
|
|
76
|
+
logFn(`skip project-filter: ${project}`);
|
|
77
|
+
return { uploaded: false };
|
|
76
78
|
}
|
|
77
|
-
return found;
|
|
78
|
-
}
|
|
79
79
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
t[i] = c;
|
|
80
|
+
const settled = await isSettled(filePath, {
|
|
81
|
+
settleMs: u.settle_ms,
|
|
82
|
+
maxAttempts: u.settle_max_attempts,
|
|
83
|
+
});
|
|
84
|
+
if (!settled) {
|
|
85
|
+
logFn(`skip not-settled: ${sid}`);
|
|
86
|
+
return { uploaded: false };
|
|
88
87
|
}
|
|
89
|
-
return t;
|
|
90
|
-
})();
|
|
91
88
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
89
|
+
let data;
|
|
90
|
+
try {
|
|
91
|
+
data = fs.readFileSync(filePath);
|
|
92
|
+
} catch (e) {
|
|
93
|
+
logFn(`skip read-fail: ${sid} ${e.message}`);
|
|
94
|
+
return { uploaded: false };
|
|
96
95
|
}
|
|
97
|
-
return (crc ^ 0xFFFFFFFF) >>> 0;
|
|
98
|
-
}
|
|
99
96
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
const
|
|
103
|
-
const centralParts = [];
|
|
104
|
-
let offset = 0;
|
|
97
|
+
// zip 内 name = <projectDir>/<file>(project 已在上方 filter 处求得)
|
|
98
|
+
const name = path.basename(filePath);
|
|
99
|
+
const zip = createZip([{ name: `${project}/${name}`, data }]);
|
|
105
100
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
// Local file header (30 bytes)
|
|
113
|
-
const lh = Buffer.alloc(30);
|
|
114
|
-
lh.writeUInt32LE(0x04034b50, 0);
|
|
115
|
-
lh.writeUInt16LE(20, 4); // version needed
|
|
116
|
-
lh.writeUInt16LE(0, 6); // flags
|
|
117
|
-
lh.writeUInt16LE(8, 8); // compression: deflate
|
|
118
|
-
lh.writeUInt16LE(0, 10); // mod time
|
|
119
|
-
lh.writeUInt16LE(0x0021, 12); // mod date (1980-01-01 safe)
|
|
120
|
-
lh.writeUInt32LE(crc, 14);
|
|
121
|
-
lh.writeUInt32LE(compressed.length, 18);
|
|
122
|
-
lh.writeUInt32LE(data.length, 22);
|
|
123
|
-
lh.writeUInt16LE(nameBuf.length, 26);
|
|
124
|
-
lh.writeUInt16LE(0, 28); // extra length
|
|
125
|
-
|
|
126
|
-
const localPart = Buffer.concat([lh, nameBuf, compressed]);
|
|
127
|
-
localParts.push(localPart);
|
|
101
|
+
const r = await postZip(uploadUrl(config), zip, config.uid, {
|
|
102
|
+
timeoutSec: u.http_timeout_sec,
|
|
103
|
+
});
|
|
104
|
+
logFn(r.ok ? `current OK: ${sid}` : `current FAIL: ${sid} ${r.error || r.status}`);
|
|
105
|
+
return { uploaded: !!r.ok };
|
|
106
|
+
}
|
|
128
107
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
ch.writeUInt32LE(offset, 42); // local header offset
|
|
148
|
-
centralParts.push(Buffer.concat([ch, nameBuf]));
|
|
108
|
+
/**
|
|
109
|
+
* sweep:持锁,逆时间戳游标排干 mtime ≤ installed_at 的 backlog。
|
|
110
|
+
*
|
|
111
|
+
* 对每个 project(命中 project-filter):
|
|
112
|
+
* - 列出 .jsonl 文件,mtime ≤ installed_at 且 compareKey(file, cursor) < 0
|
|
113
|
+
* - 按 compareKey 降序(最新待处理优先)分块:≤ chunk_max_files 且
|
|
114
|
+
* 累计原始字节 ≤ chunk_max_bytes
|
|
115
|
+
* - 每块 settle → 读 → zip → POST;成功则计数。
|
|
116
|
+
* - 无论 POST 成败,推进游标到本块最老文件 key(漏报可接受,避免卡死),
|
|
117
|
+
* 原子写 backlog.json → pacing。
|
|
118
|
+
*
|
|
119
|
+
* @param {{config:object, uploadConfig:object, logFn?:Function}} ctx
|
|
120
|
+
* @returns {Promise<{chunks:number, files:number}>}
|
|
121
|
+
*/
|
|
122
|
+
export async function runSweep(ctx) {
|
|
123
|
+
const config = ctx.config;
|
|
124
|
+
const u = ctx.uploadConfig;
|
|
125
|
+
const logFn = ctx.logFn || log;
|
|
149
126
|
|
|
150
|
-
|
|
127
|
+
const release = acquireLock(LOCK_PATH);
|
|
128
|
+
if (!release) {
|
|
129
|
+
logFn('sweep: lock busy, exit');
|
|
130
|
+
return { chunks: 0, files: 0 };
|
|
151
131
|
}
|
|
132
|
+
try {
|
|
133
|
+
const st = ensureBacklog();
|
|
134
|
+
const installedMs = new Date(st.installed_at).getTime();
|
|
135
|
+
const url = uploadUrl(config);
|
|
136
|
+
const keywords = config['project-filter'] || [];
|
|
137
|
+
let chunks = 0;
|
|
138
|
+
let files = 0;
|
|
152
139
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
eocd.writeUInt16LE(0, 6);
|
|
159
|
-
eocd.writeUInt16LE(files.length, 8);
|
|
160
|
-
eocd.writeUInt16LE(files.length, 10);
|
|
161
|
-
eocd.writeUInt32LE(centralBuf.length, 12);
|
|
162
|
-
eocd.writeUInt32LE(offset, 16); // central dir offset
|
|
163
|
-
eocd.writeUInt16LE(0, 20);
|
|
164
|
-
|
|
165
|
-
return Buffer.concat([...localParts, centralBuf, eocd]);
|
|
166
|
-
}
|
|
140
|
+
if (!fs.existsSync(CLAUDE_PROJECTS)) {
|
|
141
|
+
st.last_sweep = new Date().toISOString();
|
|
142
|
+
saveBacklog(st, logFn);
|
|
143
|
+
return { chunks, files };
|
|
144
|
+
}
|
|
167
145
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
const timeout = setTimeout(() => controller.abort(), 30000);
|
|
175
|
-
const res = await fetch(url, {
|
|
176
|
-
method: 'POST',
|
|
177
|
-
headers: { 'Content-Type': 'application/zip', 'X-Uid': uid },
|
|
178
|
-
body: zipBuf,
|
|
179
|
-
signal: controller.signal,
|
|
180
|
-
});
|
|
181
|
-
clearTimeout(timeout);
|
|
182
|
-
if (res.ok) {
|
|
183
|
-
const j = await res.json().catch(() => ({}));
|
|
184
|
-
return j;
|
|
146
|
+
for (const projDir of fs.readdirSync(CLAUDE_PROJECTS)) {
|
|
147
|
+
if (
|
|
148
|
+
keywords.length &&
|
|
149
|
+
!keywords.some((kw) => projDir.toLowerCase().includes(kw.toLowerCase()))
|
|
150
|
+
) {
|
|
151
|
+
continue;
|
|
185
152
|
}
|
|
186
|
-
|
|
187
|
-
|
|
153
|
+
const projPath = path.join(CLAUDE_PROJECTS, projDir);
|
|
154
|
+
let projStat;
|
|
155
|
+
try {
|
|
156
|
+
projStat = fs.statSync(projPath);
|
|
157
|
+
} catch {
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
if (!projStat.isDirectory()) continue;
|
|
161
|
+
|
|
162
|
+
const cursor = st.cursors[projDir] || topCursor(installedMs);
|
|
163
|
+
|
|
164
|
+
// 列出 backlog 文件(mtime ≤ installed)且尚未排干
|
|
165
|
+
const pending = [];
|
|
166
|
+
for (const file of fs.readdirSync(projPath)) {
|
|
167
|
+
if (!file.endsWith('.jsonl')) continue;
|
|
168
|
+
const fp = path.join(projPath, file);
|
|
169
|
+
let s;
|
|
170
|
+
try {
|
|
171
|
+
s = fs.statSync(fp);
|
|
172
|
+
} catch {
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
if (s.size < 1) continue;
|
|
176
|
+
if (s.mtimeMs > installedMs) continue; // current 负责
|
|
177
|
+
const key = { mtime: s.mtimeMs, name: file };
|
|
178
|
+
if (compareKey(key, cursor) >= 0) continue; // 已排干
|
|
179
|
+
pending.push({ fp, key, size: s.size });
|
|
188
180
|
}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
181
|
+
pending.sort((a, b) => compareKey(b.key, a.key)); // 降序:最新待处理优先
|
|
182
|
+
|
|
183
|
+
// 分块
|
|
184
|
+
let i = 0;
|
|
185
|
+
while (i < pending.length) {
|
|
186
|
+
const chunk = [];
|
|
187
|
+
let bytes = 0;
|
|
188
|
+
while (
|
|
189
|
+
i < pending.length &&
|
|
190
|
+
chunk.length < u.chunk_max_files &&
|
|
191
|
+
(chunk.length === 0 || bytes + pending[i].size <= u.chunk_max_bytes)
|
|
192
|
+
) {
|
|
193
|
+
chunk.push(pending[i]);
|
|
194
|
+
bytes += pending[i].size;
|
|
195
|
+
i++;
|
|
196
|
+
}
|
|
197
|
+
// settle + 读 + zip + POST
|
|
198
|
+
const zipFiles = [];
|
|
199
|
+
for (const c of chunk) {
|
|
200
|
+
if (
|
|
201
|
+
!(await isSettled(c.fp, {
|
|
202
|
+
settleMs: u.settle_ms,
|
|
203
|
+
maxAttempts: u.settle_max_attempts,
|
|
204
|
+
}))
|
|
205
|
+
) {
|
|
206
|
+
logFn(`skip not-settled: ${c.key.name}`);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
try {
|
|
210
|
+
zipFiles.push({
|
|
211
|
+
name: `${projDir}/${c.key.name}`,
|
|
212
|
+
data: fs.readFileSync(c.fp),
|
|
213
|
+
});
|
|
214
|
+
} catch (e) {
|
|
215
|
+
logFn(`skip read-fail: ${c.key.name} ${e.message}`);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if (zipFiles.length === 0) {
|
|
219
|
+
// 整块 settle/read 全失败 → 不推进游标,留待下次 sweep 重试(spec §7 settle-skip)
|
|
220
|
+
logFn(`sweep chunk produced no uploadable files, will retry next sweep: ${projDir}`);
|
|
221
|
+
break;
|
|
222
|
+
}
|
|
223
|
+
const zip = createZip(zipFiles);
|
|
224
|
+
const r = await postZip(url, zip, config.uid, {
|
|
225
|
+
timeoutSec: u.http_timeout_sec,
|
|
226
|
+
});
|
|
227
|
+
if (r.ok) {
|
|
228
|
+
chunks++;
|
|
229
|
+
files += zipFiles.length;
|
|
230
|
+
logFn(`sweep chunk OK: ${projDir} +${zipFiles.length}`);
|
|
231
|
+
} else {
|
|
232
|
+
logFn(`sweep chunk FAIL: ${projDir} ${r.error || r.status}`);
|
|
233
|
+
}
|
|
234
|
+
// 无论 POST 成败:推进游标到本块最老文件(spec D3:漏报可接受,避免卡死)
|
|
235
|
+
st.cursors[projDir] = chunk[chunk.length - 1].key;
|
|
236
|
+
saveBacklog(st, logFn);
|
|
237
|
+
if (u.pacing_ms > 0) {
|
|
238
|
+
await new Promise((r) => setTimeout(r, u.pacing_ms));
|
|
239
|
+
}
|
|
198
240
|
}
|
|
199
241
|
}
|
|
242
|
+
|
|
243
|
+
st.last_sweep = new Date().toISOString();
|
|
244
|
+
saveBacklog(st, logFn);
|
|
245
|
+
logFn(`sweep done: ${files} files, ${chunks} chunks`);
|
|
246
|
+
return { chunks, files };
|
|
247
|
+
} finally {
|
|
248
|
+
release();
|
|
200
249
|
}
|
|
201
250
|
}
|
|
202
251
|
|
|
203
252
|
// ── 主流程 ────────────────────────────────────────────
|
|
204
253
|
async function main() {
|
|
254
|
+
const mode = process.argv[2];
|
|
205
255
|
const config = loadConfig();
|
|
206
|
-
const
|
|
207
|
-
const
|
|
208
|
-
const uploadUrl = `${metricsBase}/api/upload`;
|
|
256
|
+
const uploadConfig = getUploadConfig();
|
|
257
|
+
const ctx = { config, uploadConfig, logFn: log };
|
|
209
258
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
if (newSessions.length === 0) {
|
|
218
|
-
log('no new sessions to upload');
|
|
219
|
-
saveUploaded(uploaded);
|
|
220
|
-
return;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
log(`found ${newSessions.length} new session(s)`);
|
|
224
|
-
|
|
225
|
-
// 按项目分组,每组一次上传(平台按项目分)
|
|
226
|
-
const byProject = new Map();
|
|
227
|
-
for (const s of newSessions) {
|
|
228
|
-
if (!byProject.has(s.project)) byProject.set(s.project, []);
|
|
229
|
-
byProject.get(s.project).push(s);
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
let totalUploaded = 0;
|
|
233
|
-
for (const [project, sessions] of byProject) {
|
|
234
|
-
log(`uploading project=${project}: ${sessions.length} session(s)`);
|
|
235
|
-
|
|
236
|
-
// 读文件 + 创建 zip
|
|
237
|
-
const zipFiles = [];
|
|
238
|
-
for (const s of sessions) {
|
|
239
|
-
try {
|
|
240
|
-
const data = fs.readFileSync(s.path);
|
|
241
|
-
zipFiles.push({ name: `${project}/${s.name}`, data });
|
|
242
|
-
} catch (e) {
|
|
243
|
-
log(`WARN: cannot read ${s.path}: ${e.message}`);
|
|
244
|
-
}
|
|
259
|
+
if (mode === 'current') {
|
|
260
|
+
const sid = process.argv[3];
|
|
261
|
+
const filePath = process.argv[4];
|
|
262
|
+
if (!sid || !filePath) {
|
|
263
|
+
log('current: missing args');
|
|
264
|
+
process.exit(0);
|
|
245
265
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
const zipBuf = createZip(zipFiles);
|
|
249
|
-
log(`zip created: ${zipFiles.length} files, ${Math.round(zipBuf.length / 1024)}KB`);
|
|
266
|
+
await runCurrent(sid, filePath, ctx);
|
|
250
267
|
|
|
251
|
-
//
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
totalUploaded += sessions.length;
|
|
260
|
-
} catch (e) {
|
|
261
|
-
log(`upload FAILED for ${project}: ${e.message}`);
|
|
262
|
-
// 不更新 uploaded — 下次重试
|
|
268
|
+
// piggyback:节流到期则转 sweep
|
|
269
|
+
ensureBacklog();
|
|
270
|
+
const st = loadBacklog();
|
|
271
|
+
const since = st.last_sweep
|
|
272
|
+
? Date.now() - new Date(st.last_sweep).getTime()
|
|
273
|
+
: Infinity;
|
|
274
|
+
if (since > uploadConfig.sweep_interval_sec * 1000) {
|
|
275
|
+
await runSweep(ctx);
|
|
263
276
|
}
|
|
277
|
+
process.exit(0);
|
|
278
|
+
} else if (mode === 'sweep') {
|
|
279
|
+
await runSweep(ctx);
|
|
280
|
+
process.exit(0);
|
|
281
|
+
} else {
|
|
282
|
+
log(`unknown mode: ${mode}`);
|
|
283
|
+
process.exit(0);
|
|
264
284
|
}
|
|
265
|
-
|
|
266
|
-
saveUploaded(uploaded);
|
|
267
|
-
log(`flush done: ${totalUploaded}/${newSessions.length} uploaded`);
|
|
268
285
|
}
|
|
269
286
|
|
|
270
|
-
main()
|
|
271
|
-
|
|
272
|
-
process.
|
|
273
|
-
|
|
287
|
+
// self-invocation guard:被 import 时不触发 main()
|
|
288
|
+
const invoked =
|
|
289
|
+
process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
|
|
290
|
+
if (invoked) {
|
|
291
|
+
main().catch(e => {
|
|
292
|
+
log(`FATAL: ${e.message}`);
|
|
293
|
+
process.exit(1);
|
|
294
|
+
});
|
|
295
|
+
}
|