atris 3.82.10 → 3.82.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/atris.js +14 -0
- package/commands/memory-live.js +373 -0
- package/commands/memory.js +205 -0
- package/commands/security-review.js +23 -0
- package/commands/sentinel-fix.js +8 -0
- package/commands/sentinel-powers.js +25 -4
- package/commands/sentinel-record.js +139 -0
- package/lib/known-commands.js +1 -1
- package/lib/memory-files.js +226 -0
- package/lib/memory-folder.js +602 -0
- package/lib/memory-live.js +356 -0
- package/lib/sentinel-cloud.js +1 -0
- package/lib/sentinel-record.js +877 -0
- package/lib/workspace-safety.js +1 -0
- package/package.json +1 -1
package/commands/sentinel-fix.js
CHANGED
|
@@ -222,6 +222,14 @@ async function fixCommand(argv, deps = {}) {
|
|
|
222
222
|
}
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
+
{
|
|
226
|
+
const rec = require('../lib/sentinel-record');
|
|
227
|
+
const o = { recordHome: deps.recordHome, homeDir: deps.homeDir, env: deps.env, record: deps.record };
|
|
228
|
+
let r;
|
|
229
|
+
try { r = rec.appendSafe(root, results.map((x) => rec.entryFromFix(x.record, x.res)), o); } catch (e) { r = { ok: false, reason: e.message }; }
|
|
230
|
+
const w = rec.warnLine(r);
|
|
231
|
+
if (w) console.log(w);
|
|
232
|
+
}
|
|
225
233
|
const done = results.filter((x) => x.res.ok);
|
|
226
234
|
if (done.length) {
|
|
227
235
|
const events = require('../lib/sentinel-events');
|
|
@@ -78,6 +78,22 @@ function startUpdateNotice(argv = [], { env = process.env, check, timeoutMs = 15
|
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
// ---------------------------------------------------------------- record
|
|
82
|
+
|
|
83
|
+
// Each check adds a signed entry to the security record
|
|
84
|
+
// (lib/sentinel-record.js). A failure is a warning line, never an exit code.
|
|
85
|
+
function addToRecord(root, build, deps = {}) {
|
|
86
|
+
const rec = require('../lib/sentinel-record');
|
|
87
|
+
const o = { recordHome: deps.recordHome, homeDir: deps.homeDir, env: deps.env, record: deps.record };
|
|
88
|
+
if (deps.record === false) return { ok: true, skipped: true };
|
|
89
|
+
try { return rec.appendSafe(root, build(rec, o), o); } catch (e) { return { ok: false, reason: String(e.message || e).split('\n')[0] }; }
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function recordWarn(res, json) {
|
|
93
|
+
const w = require('../lib/sentinel-record').warnLine(res);
|
|
94
|
+
if (w) (json ? console.error : console.log)(w);
|
|
95
|
+
}
|
|
96
|
+
|
|
81
97
|
// ---------------------------------------------------------------- agents
|
|
82
98
|
|
|
83
99
|
async function runAgents(root, { home = false, upload = true, deps = {} } = {}) {
|
|
@@ -90,7 +106,8 @@ async function runAgents(root, { home = false, upload = true, deps = {} } = {})
|
|
|
90
106
|
try {
|
|
91
107
|
res = await events.reportFindings(root, 'agent_check', evs, { token: upload ? tokenFor(deps) : '', base: deps.base, env: deps.env || process.env, upload });
|
|
92
108
|
} catch (e) { res = { ok: false, reason: e.message }; }
|
|
93
|
-
|
|
109
|
+
const record = addToRecord(root, (rec, o) => rec.entryFromAgents(root, result, o), deps);
|
|
110
|
+
return { result, events: evs, upload: res, record };
|
|
94
111
|
}
|
|
95
112
|
|
|
96
113
|
async function agentsCommand(argv, deps = {}) {
|
|
@@ -98,7 +115,8 @@ async function agentsCommand(argv, deps = {}) {
|
|
|
98
115
|
try { opts = parseFlags(argv, ['home', 'json', 'no-upload']); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
99
116
|
let root;
|
|
100
117
|
try { root = resolveRoot(opts, deps); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
101
|
-
const { result, upload } = await runAgents(root, { home: Boolean(opts.home), upload: !opts.noUpload, deps });
|
|
118
|
+
const { result, upload, record } = await runAgents(root, { home: Boolean(opts.home), upload: !opts.noUpload, deps });
|
|
119
|
+
recordWarn(record, opts.json);
|
|
102
120
|
if (opts.json) {
|
|
103
121
|
console.log(JSON.stringify({ ok: result.grade !== 'fail', grade: result.grade, areas: result.areas, files: result.files, findings: result.findings, upload }, null, 2));
|
|
104
122
|
} else {
|
|
@@ -153,7 +171,8 @@ async function runWatchCheck(root, { upload = true, deps = {} } = {}) {
|
|
|
153
171
|
res.heldBranch = why.branch;
|
|
154
172
|
} catch (e) { res = { ok: false, reason: e.message }; }
|
|
155
173
|
}
|
|
156
|
-
|
|
174
|
+
const record = addToRecord(root, (rec, o) => rec.entryFromWatch(root, result, { ...o, branch }), deps);
|
|
175
|
+
return { result, events: evs, upload: res, branch, record };
|
|
157
176
|
}
|
|
158
177
|
|
|
159
178
|
function heldLines(upload, branch) {
|
|
@@ -186,7 +205,8 @@ async function watchCommand(argv, deps = {}) {
|
|
|
186
205
|
let root;
|
|
187
206
|
try { root = resolveRoot(opts, deps); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
188
207
|
const watch = require('../lib/sentinel-watch');
|
|
189
|
-
const { result, upload, branch } = await runWatchCheck(root, { upload: !opts.noUpload, deps });
|
|
208
|
+
const { result, upload, branch, record } = await runWatchCheck(root, { upload: !opts.noUpload, deps });
|
|
209
|
+
recordWarn(record, opts.json);
|
|
190
210
|
let fix = null;
|
|
191
211
|
if (opts.fix && !result.offline && result.affected.length) {
|
|
192
212
|
try { fix = watch.applyFix(root, result, { run: deps.run, allowMajor: Boolean(opts.allowMajor) }); } catch (e) { fix = { ok: false, reason: String(e.message || e).split('\n')[0] }; }
|
|
@@ -279,6 +299,7 @@ async function guardCommand(argv, deps = {}) {
|
|
|
279
299
|
const blocks = inst.recentBlocks({ env });
|
|
280
300
|
const events = require('../lib/sentinel-events');
|
|
281
301
|
const waiting = events.queueSize(env);
|
|
302
|
+
recordWarn(addToRecord(root, (rec, o) => rec.entryFromGuard(root, o), { ...deps, env }), opts.json);
|
|
282
303
|
if (opts.json) { console.log(JSON.stringify({ ...st, blocked_today: blocks.length, waiting_to_send: waiting }, null, 2)); return 0; }
|
|
283
304
|
console.log('');
|
|
284
305
|
if (st.installed) console.log(` the sentinel guard is on for ${where}`);
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// atris sentinel record: the signed security record a reviewer can check.
|
|
4
|
+
//
|
|
5
|
+
// atris sentinel record [show] [path] plain summary
|
|
6
|
+
// atris sentinel record key [path] public key and fingerprint
|
|
7
|
+
// atris sentinel record export [path] --out FILE [--json]
|
|
8
|
+
// atris sentinel record verify FILE offline check of an export
|
|
9
|
+
// atris sentinel record backfill [path] add past review runs, marked as added later
|
|
10
|
+
//
|
|
11
|
+
// The guarantee and its limits sit at the top of lib/sentinel-record.js.
|
|
12
|
+
|
|
13
|
+
const fs = require('fs');
|
|
14
|
+
const path = require('path');
|
|
15
|
+
|
|
16
|
+
function help() {
|
|
17
|
+
console.log(`
|
|
18
|
+
atris sentinel record, a signed security record you can hand to a reviewer
|
|
19
|
+
|
|
20
|
+
atris sentinel record [path] what sentinel checked, what is open and fixed
|
|
21
|
+
atris sentinel record key [path] this project's public key and its fingerprint
|
|
22
|
+
atris sentinel record export [path] --out record.html
|
|
23
|
+
one html file a reviewer opens and checks in the browser
|
|
24
|
+
atris sentinel record export [path] --out record.json --json
|
|
25
|
+
atris sentinel record verify FILE check an exported record offline
|
|
26
|
+
atris sentinel record backfill [path] add past reviews, marked as added later with their dates
|
|
27
|
+
|
|
28
|
+
every review, recheck, agent check, watch, guard status and fix adds an
|
|
29
|
+
entry. each entry holds the hash of the one before it and is signed with
|
|
30
|
+
a key kept in ~/.atris. an edited, moved or removed entry shows.
|
|
31
|
+
|
|
32
|
+
what it proves: the record came from this key and was not changed.
|
|
33
|
+
what it does not prove: that the checks were thorough or the code is safe.
|
|
34
|
+
entries hold plain sentences, counts, file:line places and short ids only.
|
|
35
|
+
`);
|
|
36
|
+
return 0;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function parse(argv) {
|
|
40
|
+
const opts = { paths: [] };
|
|
41
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
42
|
+
const a = argv[i];
|
|
43
|
+
if (a === '--out') { opts.out = argv[++i]; if (!opts.out) throw new Error('--out needs a file'); }
|
|
44
|
+
else if (a === '--json') opts.json = true;
|
|
45
|
+
else if (a.startsWith('-')) throw new Error(`unknown flag: ${a}`);
|
|
46
|
+
else opts.paths.push(a);
|
|
47
|
+
}
|
|
48
|
+
return opts;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function rootOf(opts, deps) {
|
|
52
|
+
const root = path.resolve(opts.paths[0] || deps.root || process.cwd());
|
|
53
|
+
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) throw new Error(`${root} is not a folder`);
|
|
54
|
+
return root;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function libOpts(deps) {
|
|
58
|
+
const o = {};
|
|
59
|
+
if (deps.recordHome) o.recordHome = deps.recordHome;
|
|
60
|
+
if (deps.homeDir) o.homeDir = deps.homeDir;
|
|
61
|
+
if (deps.env) o.env = deps.env;
|
|
62
|
+
if (deps.now) o.now = deps.now;
|
|
63
|
+
return o;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async function recordCommand(argv = [], deps = {}) {
|
|
67
|
+
const rec = require('../lib/sentinel-record');
|
|
68
|
+
const known = ['show', 'key', 'export', 'verify', 'backfill', 'help'];
|
|
69
|
+
const sub = known.includes(argv[0]) ? argv[0] : 'show';
|
|
70
|
+
const rest = known.includes(argv[0]) ? argv.slice(1) : argv;
|
|
71
|
+
if (sub === 'help' || rest.includes('--help') || rest.includes('-h')) return help();
|
|
72
|
+
let opts;
|
|
73
|
+
try { opts = parse(rest); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
74
|
+
const lo = libOpts(deps);
|
|
75
|
+
|
|
76
|
+
if (sub === 'verify') {
|
|
77
|
+
const file = opts.paths[0];
|
|
78
|
+
if (!file) { console.error(' name the exported file: atris sentinel record verify FILE'); return 2; }
|
|
79
|
+
let doc;
|
|
80
|
+
try { doc = rec.readExport(path.resolve(file)); } catch (e) { console.error(` could not read ${file}: ${e.message}`); return 2; }
|
|
81
|
+
const r = rec.verifyDoc(doc);
|
|
82
|
+
if (opts.json) { console.log(JSON.stringify(r, null, 2)); return r.ok ? 0 : 1; }
|
|
83
|
+
if (r.ok) {
|
|
84
|
+
console.log(` verified: all ${r.entries} entries for ${doc.project} are signed by key ${r.fingerprint}`);
|
|
85
|
+
console.log(' none were changed, moved or removed. compare the fingerprint with the one the company gave you');
|
|
86
|
+
return 0;
|
|
87
|
+
}
|
|
88
|
+
console.log(` broken at ${r.label}: ${r.reason}`);
|
|
89
|
+
return 1;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
let root;
|
|
93
|
+
try { root = rootOf(opts, deps); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
94
|
+
const project = rec.projectName(root);
|
|
95
|
+
const id = rec.projectId(root);
|
|
96
|
+
|
|
97
|
+
if (sub === 'key') {
|
|
98
|
+
let key;
|
|
99
|
+
try { key = rec.loadKey(id, { ...lo, name: project }); } catch (e) { console.error(` could not open the record key: ${e.message}`); return 1; }
|
|
100
|
+
if (opts.json) { console.log(JSON.stringify({ project, project_id: id, public_key: key.publicKey, fingerprint: key.fingerprint }, null, 2)); return 0; }
|
|
101
|
+
console.log(`\n record key for ${project}${key.created ? ' (made just now)' : ''}`);
|
|
102
|
+
console.log(` public key: ${key.publicKey}`);
|
|
103
|
+
console.log(` fingerprint: ${key.fingerprint}`);
|
|
104
|
+
console.log(' give the fingerprint to reviewers through a channel they trust. the private key stays in ~/.atris\n');
|
|
105
|
+
return 0;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (sub === 'backfill') {
|
|
109
|
+
let res;
|
|
110
|
+
try { res = rec.backfill(root, lo); } catch (e) { console.error(` could not backfill the record: ${e.message}`); return 1; }
|
|
111
|
+
console.log(` added ${res.added.length} past run${res.added.length === 1 ? '' : 's'} to the record, each marked as added later with its original date`);
|
|
112
|
+
if (res.skipped.length) console.log(` ${res.skipped.length} already in the record or unreadable`);
|
|
113
|
+
return 0;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (sub === 'export') {
|
|
117
|
+
if (!opts.out) { console.error(' name the output file: atris sentinel record export --out record.html'); return 2; }
|
|
118
|
+
let doc;
|
|
119
|
+
try { doc = rec.buildExport(root, lo); } catch (e) { console.error(` could not export the record: ${e.message}`); return 1; }
|
|
120
|
+
const out = path.resolve(opts.out);
|
|
121
|
+
const body = opts.json ? `${JSON.stringify(doc, null, 2)}\n` : rec.renderHtml(doc);
|
|
122
|
+
fs.mkdirSync(path.dirname(out), { recursive: true });
|
|
123
|
+
fs.writeFileSync(out, body);
|
|
124
|
+
console.log(` wrote ${doc.entries.length} signed entries to ${out}`);
|
|
125
|
+
console.log(` key fingerprint: ${doc.key_fingerprint}`);
|
|
126
|
+
console.log(opts.json ? ' check it with: atris sentinel record verify FILE' : ' a reviewer opens it in a browser. it checks itself and shows verified or broken');
|
|
127
|
+
return 0;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// show
|
|
131
|
+
let entries;
|
|
132
|
+
try { entries = rec.readEntries(rec.recordPath(root, lo)); } catch (e) { console.error(` could not read the record: ${e.message}`); return 1; }
|
|
133
|
+
const key = rec.loadKey(id, { ...lo, create: false });
|
|
134
|
+
if (opts.json) { console.log(JSON.stringify({ project, project_id: id, key_fingerprint: key ? key.fingerprint : null, ...rec.summarize(entries) }, null, 2)); return 0; }
|
|
135
|
+
console.log(rec.renderShow(project, entries, key));
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
module.exports = { recordCommand };
|
package/lib/known-commands.js
CHANGED
|
@@ -4,7 +4,7 @@ const knownCommands = ['init', 'guide', 'doctor', 'doc-health', 'install', 'work
|
|
|
4
4
|
'activate', '_activate', 'agent', 'team', 'chat', 'fast', 'ax', 'console', 'serve', 'login', 'logout', 'whoami', 'switch', 'use', 'accounts', '_resolve', '_profile-email', '_switch-session', 'shell-init', 'update', 'upgrade', 'version', 'help', 'next', 'atris',
|
|
5
5
|
'clean', 'close', 'harvest', 'verify', 'recover', 'search', 'scout', 'skill', 'member', 'codex-goal', 'app', 'apps', 'learn', 'lesson', 'taste', 'teach', 'plugin', 'experiments', 'bench', 'router', 'tree', 'receipt', 'proof', 'openclaw', 'pull', 'push', 'watch', 'cloud', 'live', 'align', 'terminal', 'computer', 'diff', 'business', 'sync', 'youtube', 'x-search',
|
|
6
6
|
'ingest', 'query', 'lint', 'loop', 'pulse', 'task', 'mission', 'decide', 'campaign', 'agents', 'probe', 'worktree', 'land', 'caretaker', 'autoland', 'drive', 'aeo', 'slop', 'voice', 'strings', 'write', 'security-review', 'secure', 'sentinel', 'deck', 'site', 'theme', 'card', 'reel', 'improve', 'study', 'rainmaker', 'xp', 'play', 'gm', 'game', 'x', 'recap', 'report', 'signup', 'clarity', 'interview', 'meet', 'moves', 'unknowns', 'avail', 'scoreboard', 'scorecard', 'rating', 'sync-checkout', 'zen',
|
|
7
|
-
'ci', 'github', 'vercel', 'supabase', 'linear', 'stripe', 'balance', 'usage', 'spend', 'api-key', 'agent-client', 'topup', 'buy', 'transactions', 'hire', 'hires', 'mail', 'design', 'mcp', 'gmail', 'google', 'calendar', 'twitter', 'slack', 'imessage', 'integrations', 'setup', 'clean-workspace', 'cw',
|
|
7
|
+
'ci', 'github', 'vercel', 'supabase', 'linear', 'stripe', 'memory', 'balance', 'usage', 'spend', 'api-key', 'agent-client', 'topup', 'buy', 'transactions', 'hire', 'hires', 'mail', 'design', 'mcp', 'gmail', 'google', 'calendar', 'twitter', 'slack', 'imessage', 'integrations', 'setup', 'clean-workspace', 'cw',
|
|
8
8
|
'fork', 'browse', 'publish', 'pack', 'sleep', 'wake', 'feedback', 'errors', 'wiki', 'code-review', 'cr', 'soul', 'fleet', 'fleet-report', 'loops', 'self-improve', 'compile', 'spaceship', 'truth', 'sign', 'engine', 'engines', 'link', 'playbook', 'feed', 'brief', 'social', 'host', 'people', 'follow', 'unfollow', 'friends', 'msg', 'inbox', 'invite', 'join', 'rsi', 'pay', 'trust'];
|
|
9
9
|
|
|
10
10
|
// Damerau-Levenshtein edit distance between two short strings. Counts an
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Real files in an Atris memory folder: photos, PDFs, recordings, anything
|
|
4
|
+
// under files/. The bytes go straight between this laptop and the private
|
|
5
|
+
// bucket, on short-lived links the Atris server signs for one file at a time.
|
|
6
|
+
// They never pass through the server, and no file is ever held whole in
|
|
7
|
+
// memory here: hashing, sending and fetching all stream in 1 MB pieces.
|
|
8
|
+
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const crypto = require('crypto');
|
|
12
|
+
const http = require('http');
|
|
13
|
+
const https = require('https');
|
|
14
|
+
const { Transform, pipeline } = require('stream');
|
|
15
|
+
|
|
16
|
+
// Pages the server writes and reads as text, up to its 1 MB write limit.
|
|
17
|
+
// Everything else (any other type, or bigger) goes as a file.
|
|
18
|
+
const TEXT_ENDINGS = ['.md', '.txt', '.json', '.jsonl', '.csv', '.yaml', '.yml'];
|
|
19
|
+
const TEXT_MAX_BYTES = 1000000;
|
|
20
|
+
const LARGE_BYTES = 4 * 1024 * 1024; // files this big move one at a time, with a progress line
|
|
21
|
+
const PIECE = 1024 * 1024;
|
|
22
|
+
const IDLE_MS = 60000;
|
|
23
|
+
|
|
24
|
+
function goesAsFile(rel, size) {
|
|
25
|
+
const lower = String(rel).toLowerCase();
|
|
26
|
+
return !(Number(size) <= TEXT_MAX_BYTES) || !TEXT_ENDINGS.some((ending) => lower.endsWith(ending));
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function printOf(buffer) {
|
|
30
|
+
return {
|
|
31
|
+
sha256: crypto.createHash('sha256').update(buffer).digest('hex'),
|
|
32
|
+
md5: crypto.createHash('md5').update(buffer).digest('hex'),
|
|
33
|
+
bytes: buffer.length,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// The fingerprints of a file on disk, read 1 MB at a time.
|
|
38
|
+
function hashFileSync(abs) {
|
|
39
|
+
const sha = crypto.createHash('sha256');
|
|
40
|
+
const md5 = crypto.createHash('md5');
|
|
41
|
+
const piece = Buffer.allocUnsafe(PIECE);
|
|
42
|
+
const fd = fs.openSync(abs, 'r');
|
|
43
|
+
let bytes = 0;
|
|
44
|
+
try {
|
|
45
|
+
for (;;) {
|
|
46
|
+
const n = fs.readSync(fd, piece, 0, PIECE, null);
|
|
47
|
+
if (!n) break;
|
|
48
|
+
sha.update(piece.subarray(0, n));
|
|
49
|
+
md5.update(piece.subarray(0, n));
|
|
50
|
+
bytes += n;
|
|
51
|
+
}
|
|
52
|
+
} finally {
|
|
53
|
+
fs.closeSync(fd);
|
|
54
|
+
}
|
|
55
|
+
return { sha256: sha.digest('hex'), md5: md5.digest('hex'), bytes };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function size(n) {
|
|
59
|
+
const units = [['GB', 1024 ** 3], ['MB', 1024 ** 2], ['KB', 1024]];
|
|
60
|
+
for (const [unit, step] of units) {
|
|
61
|
+
if (n >= step) return `${(n / step).toFixed(n % step === 0 || n >= 10 * step ? 0 : 1)} ${unit}`;
|
|
62
|
+
}
|
|
63
|
+
return `${n} bytes`;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Links come from the Atris server. Only https, or plain http to this machine (tests, a local server).
|
|
67
|
+
function transportFor(url) {
|
|
68
|
+
const parsed = new URL(url);
|
|
69
|
+
if (parsed.protocol === 'https:') return { parsed, transport: https };
|
|
70
|
+
if (parsed.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(parsed.hostname)) {
|
|
71
|
+
return { parsed, transport: http };
|
|
72
|
+
}
|
|
73
|
+
throw new Error('the Atris server sent a link that is not https');
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// What S3 said, without the link (it carries a signature).
|
|
77
|
+
function s3Said(status, body) {
|
|
78
|
+
const code = /<Code>([^<]{1,80})<\/Code>/.exec(body);
|
|
79
|
+
const message = /<Message>([^<]{1,200})<\/Message>/.exec(body);
|
|
80
|
+
if (code) return `storage refused it (${code[1]}${message ? `: ${message[1]}` : ''})`;
|
|
81
|
+
return `storage answered ${status}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function readSmall(res, done) {
|
|
85
|
+
const chunks = [];
|
|
86
|
+
let kept = 0;
|
|
87
|
+
res.on('data', (chunk) => { if (kept < 16384) { chunks.push(chunk); kept += chunk.length; } });
|
|
88
|
+
res.on('end', () => done(Buffer.concat(chunks).toString('utf8')));
|
|
89
|
+
res.on('error', () => done(''));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// POST one file to a signed S3 upload form: every field the server gave, the file last.
|
|
93
|
+
// Resolves with the fingerprints of exactly the bytes sent.
|
|
94
|
+
function sendFile({ url, fields, abs, bytes: expected, onProgress = null }) {
|
|
95
|
+
const { parsed, transport } = transportFor(url);
|
|
96
|
+
const boundary = `----atris-${crypto.randomBytes(16).toString('hex')}`;
|
|
97
|
+
let head = '';
|
|
98
|
+
for (const [name, value] of Object.entries(fields || {})) {
|
|
99
|
+
if (!/^[A-Za-z0-9_-]{1,100}$/.test(name)) throw new Error('the upload form has a field this tool does not send');
|
|
100
|
+
head += `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`;
|
|
101
|
+
}
|
|
102
|
+
head += `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="file"\r\n`
|
|
103
|
+
+ 'Content-Type: application/octet-stream\r\n\r\n';
|
|
104
|
+
const start = Buffer.from(head, 'utf8');
|
|
105
|
+
const end = Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8');
|
|
106
|
+
|
|
107
|
+
return new Promise((resolve, reject) => {
|
|
108
|
+
let settled = false;
|
|
109
|
+
let sent = null;
|
|
110
|
+
const finish = (error, value) => {
|
|
111
|
+
if (settled) return;
|
|
112
|
+
settled = true;
|
|
113
|
+
if (error) reject(error); else resolve(value);
|
|
114
|
+
};
|
|
115
|
+
const req = transport.request(parsed, {
|
|
116
|
+
method: 'POST',
|
|
117
|
+
headers: {
|
|
118
|
+
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
|
119
|
+
'Content-Length': start.length + expected + end.length,
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
req.setTimeout(IDLE_MS, () => req.destroy(new Error('the upload stalled for a minute')));
|
|
123
|
+
req.on('error', (error) => finish(error));
|
|
124
|
+
req.on('response', (res) => readSmall(res, (body) => {
|
|
125
|
+
if (res.statusCode >= 200 && res.statusCode < 300 && sent) finish(null, sent);
|
|
126
|
+
else finish(new Error(s3Said(res.statusCode, body)));
|
|
127
|
+
}));
|
|
128
|
+
const drained = () => new Promise((go) => {
|
|
129
|
+
const on = () => { req.off('drain', on); req.off('close', on); go(); };
|
|
130
|
+
req.on('drain', on);
|
|
131
|
+
req.on('close', on);
|
|
132
|
+
});
|
|
133
|
+
(async () => {
|
|
134
|
+
const sha = crypto.createHash('sha256');
|
|
135
|
+
const md5 = crypto.createHash('md5');
|
|
136
|
+
let count = 0;
|
|
137
|
+
const stream = fs.createReadStream(abs, { highWaterMark: PIECE });
|
|
138
|
+
req.write(start);
|
|
139
|
+
try {
|
|
140
|
+
for await (const chunk of stream) {
|
|
141
|
+
if (settled || req.destroyed) return;
|
|
142
|
+
count += chunk.length;
|
|
143
|
+
if (count > expected) throw new Error('the file changed while it was being sent');
|
|
144
|
+
sha.update(chunk);
|
|
145
|
+
md5.update(chunk);
|
|
146
|
+
if (onProgress) onProgress(count, expected);
|
|
147
|
+
if (!req.write(chunk)) await drained();
|
|
148
|
+
}
|
|
149
|
+
} finally {
|
|
150
|
+
stream.destroy();
|
|
151
|
+
}
|
|
152
|
+
if (count !== expected) throw new Error('the file changed while it was being sent');
|
|
153
|
+
sent = { sha256: sha.digest('hex'), md5: md5.digest('hex'), bytes: count };
|
|
154
|
+
req.end(end);
|
|
155
|
+
})().catch((error) => req.destroy(error));
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// GET a signed download link into `temp`, streaming. Resolves with the fingerprints of what arrived.
|
|
160
|
+
function fetchFile({ url, temp, bytes: expected = 0, onProgress = null }) {
|
|
161
|
+
const { parsed, transport } = transportFor(url);
|
|
162
|
+
return new Promise((resolve, reject) => {
|
|
163
|
+
const req = transport.get(parsed, (res) => {
|
|
164
|
+
if (res.statusCode !== 200) {
|
|
165
|
+
readSmall(res, (body) => reject(new Error(s3Said(res.statusCode, body))));
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
const sha = crypto.createHash('sha256');
|
|
169
|
+
const md5 = crypto.createHash('md5');
|
|
170
|
+
let count = 0;
|
|
171
|
+
const tap = new Transform({
|
|
172
|
+
transform(chunk, _encoding, next) {
|
|
173
|
+
count += chunk.length;
|
|
174
|
+
sha.update(chunk);
|
|
175
|
+
md5.update(chunk);
|
|
176
|
+
if (onProgress) onProgress(count, expected);
|
|
177
|
+
next(null, chunk);
|
|
178
|
+
},
|
|
179
|
+
});
|
|
180
|
+
pipeline(res, tap, fs.createWriteStream(temp, { flags: 'wx' }), (error) => {
|
|
181
|
+
if (error) reject(error);
|
|
182
|
+
else resolve({ sha256: sha.digest('hex'), md5: md5.digest('hex'), bytes: count });
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
req.setTimeout(IDLE_MS, () => req.destroy(new Error('the download stalled for a minute')));
|
|
186
|
+
req.on('error', reject);
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// A hidden name beside the target, so a half-fetched file is never mistaken for the real one.
|
|
191
|
+
function tempBeside(abs) {
|
|
192
|
+
return path.join(path.dirname(abs), `.${path.basename(abs)}.atris-tmp-${process.pid}-${crypto.randomBytes(4).toString('hex')}`);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// One line per large file. On a terminal it updates in place with a percent;
|
|
196
|
+
// otherwise it is printed once, when the file starts moving.
|
|
197
|
+
function progressPrinter({ log, tty = null }) {
|
|
198
|
+
const shown = new Map();
|
|
199
|
+
return ({ rel, verb, done, total, finished = false }) => {
|
|
200
|
+
if (total < LARGE_BYTES) return;
|
|
201
|
+
const label = `${verb} ${rel} (${size(total)})`;
|
|
202
|
+
if (!tty) {
|
|
203
|
+
if (!shown.has(rel)) { shown.set(rel, true); log(`${label}...`); }
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (finished) {
|
|
207
|
+
if (shown.has(rel)) tty.write('\n');
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
const percent = Math.floor((done * 100) / Math.max(1, total));
|
|
211
|
+
if (shown.get(rel) === percent) return;
|
|
212
|
+
shown.set(rel, percent);
|
|
213
|
+
tty.write(`\r${label} ${percent}%`);
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
module.exports = {
|
|
218
|
+
LARGE_BYTES,
|
|
219
|
+
fetchFile,
|
|
220
|
+
goesAsFile,
|
|
221
|
+
hashFileSync,
|
|
222
|
+
printOf,
|
|
223
|
+
progressPrinter,
|
|
224
|
+
sendFile,
|
|
225
|
+
tempBeside,
|
|
226
|
+
};
|