altalandi 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +260 -260
- package/altalandi.mjs +330 -325
- package/auth.mjs +223 -223
- package/batch.mjs +620 -620
- package/package.json +1 -1
- package/report.mjs +5 -3
- package/skill/SKILL.md +180 -112
- package/update.mjs +313 -273
package/batch.mjs
CHANGED
|
@@ -1,620 +1,620 @@
|
|
|
1
|
-
// `altalandi <folder | url>`: refine a whole folder of documents, one web page
|
|
2
|
-
// or a site in one command, and write the Icelandic next to the original.
|
|
3
|
-
//
|
|
4
|
-
// The order of things is the safety story:
|
|
5
|
-
// 1. plan read every document and cut it into passages (segment.mjs,
|
|
6
|
-
// web.mjs); nothing is sent yet
|
|
7
|
-
// 2. estimate files, passages, characters and billing units, printed before
|
|
8
|
-
// anything is sent
|
|
9
|
-
// 3. consent a y/N question in a terminal, or --yes; with neither, nothing
|
|
10
|
-
// is sent (--dry-run stops here on purpose)
|
|
11
|
-
// 4. refine one call per passage, 2 at a time by default; a 429 waits for
|
|
12
|
-
// Retry-After, a 402 stops the run cleanly
|
|
13
|
-
// 5. write each file once all its passages are back; a passage whose
|
|
14
|
-
// meaning check did not pass keeps its original text; nothing is
|
|
15
|
-
// ever written inside the input
|
|
16
|
-
// 6. report one Markdown summary of every file and every change
|
|
17
|
-
//
|
|
18
|
-
// Every side effect is injectable (io), so the tests need no network, terminal
|
|
19
|
-
// or API key.
|
|
20
|
-
|
|
21
|
-
import { existsSync, statSync } from "node:fs";
|
|
22
|
-
import { mkdir, readdir, readFile, realpath, writeFile } from "node:fs/promises";
|
|
23
|
-
import path from "node:path";
|
|
24
|
-
|
|
25
|
-
import { refine } from "./client.mjs";
|
|
26
|
-
import { writeBatchReport } from "./report.mjs";
|
|
27
|
-
import { assemble, FORMATS, isConfigJson, repairCodeSpans, segment, segmentMarkdown, splitForApi } from "./segment.mjs";
|
|
28
|
-
import { pagePath, readPage, readSite, safeSegment, userAgent } from "./web.mjs";
|
|
29
|
-
|
|
30
|
-
export const EXIT_OK = 0;
|
|
31
|
-
export const EXIT_ERROR = 1;
|
|
32
|
-
export const EXIT_KEPT = 2; // finished, but some passages kept their original text
|
|
33
|
-
export const EXIT_CREDIT = 4; // stopped: out of credit or allowance (as for a single text)
|
|
34
|
-
|
|
35
|
-
// altalandi-reports: the reports of earlier runs are not documents to refine.
|
|
36
|
-
const SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", "altalandi-reports"]);
|
|
37
|
-
const MAX_FILE_BYTES = 5 * 1024 * 1024;
|
|
38
|
-
const MAX_RETRY_WAIT_MS = 5 * 60_000; // a longer Retry-After means "come back later"
|
|
39
|
-
const MAX_RATE_LIMIT_RETRIES = 8;
|
|
40
|
-
const MAX_TRANSIENT_RETRIES = 2;
|
|
41
|
-
const ERRORS_IN_A_ROW_TO_STOP = 3;
|
|
42
|
-
const DOMAINS = ["general", "marketing", "data-analysis", "childrens-fiction"];
|
|
43
|
-
const REGISTERS = ["formal", "neutral", "warm"];
|
|
44
|
-
|
|
45
|
-
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
46
|
-
const fmt = (n) => Math.round(n).toLocaleString("en-US");
|
|
47
|
-
const plural = (n, one, many) => `${fmt(n)} ${n === 1 ? one : many}`;
|
|
48
|
-
const fold = (p) => (process.platform === "win32" || process.platform === "darwin" ? p.toLowerCase() : p);
|
|
49
|
-
const samePath = (a, b) => fold(path.resolve(a)) === fold(path.resolve(b));
|
|
50
|
-
const isInside = (child, parent) => {
|
|
51
|
-
const rel = path.relative(fold(path.resolve(parent)), fold(path.resolve(child)));
|
|
52
|
-
return Boolean(rel) && !rel.startsWith("..") && !path.isAbsolute(rel);
|
|
53
|
-
};
|
|
54
|
-
const display = (p, cwd) => {
|
|
55
|
-
const rel = path.relative(cwd, p);
|
|
56
|
-
return rel && !rel.startsWith("..") && !path.isAbsolute(rel) ? rel : p;
|
|
57
|
-
};
|
|
58
|
-
const preview = (s, n = 72) => {
|
|
59
|
-
const one = String(s).replace(/\s+/g, " ").trim();
|
|
60
|
-
return one.length > n ? `${one.slice(0, n - 1)}…` : one;
|
|
61
|
-
};
|
|
62
|
-
|
|
63
|
-
/** Every whitespace token is at least one billing unit; a long unbroken token
|
|
64
|
-
* is one unit per 20 characters (the server's rule, docs/api/README.md). */
|
|
65
|
-
const billingTokens = (text) => text.split(/\s+/).filter(Boolean)
|
|
66
|
-
.reduce((n, t) => n + Math.max(1, Math.ceil(t.length / 20)), 0);
|
|
67
|
-
|
|
68
|
-
/** The real path of p, or of its deepest existing ancestor plus the rest, so a
|
|
69
|
-
* symlinked --out is compared where it really points. */
|
|
70
|
-
async function realish(p) {
|
|
71
|
-
const abs = path.resolve(p);
|
|
72
|
-
const rest = [];
|
|
73
|
-
let probe = abs;
|
|
74
|
-
for (;;) {
|
|
75
|
-
try {
|
|
76
|
-
return path.join(await realpath(probe), ...rest.reverse());
|
|
77
|
-
} catch {
|
|
78
|
-
const parent = path.dirname(probe);
|
|
79
|
-
if (parent === probe) return abs;
|
|
80
|
-
rest.push(path.basename(probe));
|
|
81
|
-
probe = parent;
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
async function walk(root) {
|
|
87
|
-
const files = [];
|
|
88
|
-
const visit = async (dir, rel) => {
|
|
89
|
-
const entries = await readdir(dir, { withFileTypes: true });
|
|
90
|
-
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
91
|
-
for (const e of entries) {
|
|
92
|
-
// Hidden files and folders, and symlinks (no loops, no escaping the input).
|
|
93
|
-
if (e.name.startsWith(".") || e.isSymbolicLink()) continue;
|
|
94
|
-
const childRel = rel ? `${rel}/${e.name}` : e.name;
|
|
95
|
-
if (e.isDirectory()) {
|
|
96
|
-
if (!SKIP_DIRS.has(e.name)) await visit(path.join(dir, e.name), childRel);
|
|
97
|
-
} else if (e.isFile() && FORMATS[path.extname(e.name).toLowerCase()]) {
|
|
98
|
-
files.push(childRel);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
};
|
|
102
|
-
await visit(root, "");
|
|
103
|
-
return files;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
function usage(message) {
|
|
107
|
-
const e = new Error(message);
|
|
108
|
-
e.usage = true;
|
|
109
|
-
return e;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
// ── plans ───────────────────────────────────────────────────────────────────
|
|
113
|
-
|
|
114
|
-
async function planFolder(opts, io) {
|
|
115
|
-
const cwd = io.cwd || process.cwd();
|
|
116
|
-
const given = path.resolve(cwd, opts.target);
|
|
117
|
-
let input;
|
|
118
|
-
try { input = await realpath(given); } catch { throw usage(`No such folder: ${opts.target}`); }
|
|
119
|
-
if (!statSync(input).isDirectory()) throw usage(`${opts.target} is not a folder.`);
|
|
120
|
-
if (!path.basename(input)) throw usage("Give the output folder with --out.");
|
|
121
|
-
const output = path.resolve(cwd, opts.out || path.join(path.dirname(given), `${path.basename(given)}-is`));
|
|
122
|
-
const outputReal = await realish(output);
|
|
123
|
-
if (samePath(outputReal, input) || isInside(outputReal, input) || isInside(input, outputReal)) {
|
|
124
|
-
throw usage(`--out must be a separate folder, not the input folder, inside it or around it: ${output}`);
|
|
125
|
-
}
|
|
126
|
-
if (existsSync(output) && !statSync(output).isDirectory()) throw usage(`--out ${output} is a file; give a folder.`);
|
|
127
|
-
|
|
128
|
-
const files = [];
|
|
129
|
-
for (const rel of await walk(input)) {
|
|
130
|
-
const source = path.join(input, ...rel.split("/"));
|
|
131
|
-
const file = {
|
|
132
|
-
path: rel, source, target: path.join(output, ...rel.split("/")), format: FORMATS[path.extname(rel).toLowerCase()],
|
|
133
|
-
};
|
|
134
|
-
file.bytes = await readFile(source);
|
|
135
|
-
let text;
|
|
136
|
-
if (file.bytes.length > MAX_FILE_BYTES) file.note = "larger than 5 MB";
|
|
137
|
-
else {
|
|
138
|
-
try { text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(file.bytes); } catch { file.note = "not UTF-8 text"; }
|
|
139
|
-
}
|
|
140
|
-
if (text !== undefined) {
|
|
141
|
-
if (file.format === "json" && isConfigJson(path.basename(rel))) file.note = "configuration, not a string catalog";
|
|
142
|
-
else {
|
|
143
|
-
const seg = segment(text, file.format);
|
|
144
|
-
file.text = text;
|
|
145
|
-
file.units = seg.units;
|
|
146
|
-
file.note = seg.note || (seg.units.length ? undefined : "nothing to refine");
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
file.status = file.note ? "copy" : "refine";
|
|
150
|
-
files.push(file);
|
|
151
|
-
}
|
|
152
|
-
return { mode: "folder", label: opts.target, input, output, outputIsFolder: true, files };
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
const withSource = (page) => `<!-- Source: ${page.url} -->\n\n${page.markdown}\n`;
|
|
156
|
-
|
|
157
|
-
function pageFile(page, rel, target) {
|
|
158
|
-
const text = withSource(page);
|
|
159
|
-
const { units } = segmentMarkdown(text);
|
|
160
|
-
return {
|
|
161
|
-
path: rel, source: page.url, target, format: "md", text, units,
|
|
162
|
-
status: units.length ? "refine" : "copy", note: units.length ? undefined : "nothing to refine",
|
|
163
|
-
};
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
async function planWeb(opts, io, log) {
|
|
167
|
-
const cwd = io.cwd || process.cwd();
|
|
168
|
-
const web = { fetch: io.fetch, sleep: io.sleep, userAgent: userAgent(io.version || "0") };
|
|
169
|
-
if (!opts.site) {
|
|
170
|
-
const page = await readPage(opts.target, web);
|
|
171
|
-
const last = pagePath(page.url).replace(/\.md$/, "").split("/").pop();
|
|
172
|
-
const name = `${last === "index" ? safeSegment(new URL(page.url).host) : last}-is.md`;
|
|
173
|
-
let target = path.resolve(cwd, opts.out || name);
|
|
174
|
-
if (existsSync(target) && statSync(target).isDirectory()) target = path.join(target, name);
|
|
175
|
-
return { mode: "page", label: opts.target, output: target, outputIsFolder: false, files: [pageFile(page, path.basename(target), target)] };
|
|
176
|
-
}
|
|
177
|
-
log(`Reading ${opts.target} (up to ${plural(opts.maxPages, "page", "pages")}) …`);
|
|
178
|
-
const { pages, skipped, source } = await readSite(opts.target, {
|
|
179
|
-
maxPages: opts.maxPages, io: web, onPage: (p) => log(` read ${p.url}`),
|
|
180
|
-
});
|
|
181
|
-
const output = path.resolve(cwd, opts.out || `${safeSegment(new URL(pages[0].url).host)}-is`);
|
|
182
|
-
if (existsSync(output) && !statSync(output).isDirectory()) throw usage(`--out ${output} is a file; --site writes a folder.`);
|
|
183
|
-
const taken = new Set();
|
|
184
|
-
const files = pages.map((page) => {
|
|
185
|
-
let rel = pagePath(page.url);
|
|
186
|
-
for (let k = 2; taken.has(rel.toLowerCase()); k++) rel = pagePath(page.url).replace(/\.md$/, `-${k}.md`);
|
|
187
|
-
taken.add(rel.toLowerCase());
|
|
188
|
-
return pageFile(page, rel, path.join(output, ...rel.split("/")));
|
|
189
|
-
});
|
|
190
|
-
return { mode: "site", label: opts.target, output, outputIsFolder: true, files, skipped, discovery: source };
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
// ── calls ───────────────────────────────────────────────────────────────────
|
|
194
|
-
|
|
195
|
-
/** One call per distinct piece of text: a passage seen twice is paid once. A
|
|
196
|
-
* passage over the API limit becomes several pieces cut at sentences. */
|
|
197
|
-
function buildCalls(files, maxChars) {
|
|
198
|
-
const calls = new Map();
|
|
199
|
-
let budgetUsed = 0;
|
|
200
|
-
let overBudget = false;
|
|
201
|
-
for (const file of files) {
|
|
202
|
-
if (file.status !== "refine") continue;
|
|
203
|
-
const fresh = new Map();
|
|
204
|
-
for (const unit of file.units) {
|
|
205
|
-
const { pieces, joins } = splitForApi(unit.text);
|
|
206
|
-
unit.joins = joins;
|
|
207
|
-
unit.calls = pieces.map((text) => calls.get(text) || fresh.get(text) || fresh.set(text, { text, units: [] }).get(text));
|
|
208
|
-
}
|
|
209
|
-
const added = [...fresh.values()].reduce((n, c) => n + c.text.length, 0);
|
|
210
|
-
if (overBudget || (maxChars && budgetUsed + added > maxChars)) {
|
|
211
|
-
overBudget = true;
|
|
212
|
-
file.status = "over-budget";
|
|
213
|
-
continue;
|
|
214
|
-
}
|
|
215
|
-
budgetUsed += added;
|
|
216
|
-
for (const [text, call] of fresh) calls.set(text, call);
|
|
217
|
-
for (const unit of file.units) {
|
|
218
|
-
unit.file = file;
|
|
219
|
-
for (const call of new Set(unit.calls)) call.units.push(unit);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
return [...calls.values()];
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
async function fetchPricing(baseUrl, fetchImpl) {
|
|
226
|
-
const controller = new AbortController();
|
|
227
|
-
const timer = setTimeout(() => controller.abort(), 5000);
|
|
228
|
-
try {
|
|
229
|
-
const res = await fetchImpl(`${baseUrl}/v1/pricing`, { headers: { accept: "application/json" }, signal: controller.signal });
|
|
230
|
-
if (!res.ok) return null;
|
|
231
|
-
const p = JSON.parse(await res.text());
|
|
232
|
-
const lane = p?.tiers?.standard?.fast;
|
|
233
|
-
if (!Number.isFinite(p?.minimumBillableUnits) || !lane) return null;
|
|
234
|
-
return {
|
|
235
|
-
minUnits: p.minimumBillableUnits,
|
|
236
|
-
multiplier: Number(lane.multiplier) || 1,
|
|
237
|
-
iskPer1000: Number(lane.iskPer1000Units),
|
|
238
|
-
freeDaily: Number(p.accountFreeDailyUnits) || 0,
|
|
239
|
-
};
|
|
240
|
-
} catch {
|
|
241
|
-
return null;
|
|
242
|
-
} finally {
|
|
243
|
-
clearTimeout(timer);
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
function estimateLines(plan, calls, pricing, opts) {
|
|
248
|
-
const refining = plan.files.filter((f) => f.status === "refine");
|
|
249
|
-
const passages = refining.reduce((n, f) => n + f.units.length, 0);
|
|
250
|
-
const chars = calls.reduce((n, c) => n + c.text.length, 0);
|
|
251
|
-
const lines = [
|
|
252
|
-
`${plural(refining.length, "file", "files")}, ${plural(passages, "passage", "passages")}, about ${fmt(chars)} characters`
|
|
253
|
-
+ `${calls.length !== passages ? ` (${plural(calls.length, "call", "calls")}: repeated passages are sent once)` : ""}.`,
|
|
254
|
-
];
|
|
255
|
-
const copied = plan.files.filter((f) => f.status === "copy").length;
|
|
256
|
-
if (copied) lines.push(`${plural(copied, "more file has", "more files have")} nothing to refine and ${copied === 1 ? "is" : "are"} copied unchanged.`);
|
|
257
|
-
const over = plan.files.filter((f) => f.status === "over-budget");
|
|
258
|
-
if (over.length) {
|
|
259
|
-
lines.push(`--max-chars ${fmt(opts.maxChars)}: ${plural(over.length, "file does", "files do")} not fit and will not be sent `
|
|
260
|
-
+ `(from ${over[0].path} on).`);
|
|
261
|
-
}
|
|
262
|
-
if (pricing && calls.length) {
|
|
263
|
-
const units = calls.reduce((n, c) => n + Math.max(pricing.minUnits, billingTokens(c.text)), 0) * pricing.multiplier;
|
|
264
|
-
const isk = Number.isFinite(pricing.iskPer1000) ? `, about ${fmt((units * pricing.iskPer1000) / 1000)} kr if paid from credit` : "";
|
|
265
|
-
lines.push(`Billing: about ${fmt(units)} units (at least ${fmt(pricing.minUnits)} a call)${isk}.`
|
|
266
|
-
+ `${pricing.freeDaily ? ` The free allowance is ${fmt(pricing.freeDaily)} units a day.` : ""}`);
|
|
267
|
-
}
|
|
268
|
-
return lines;
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
// ── the run ─────────────────────────────────────────────────────────────────
|
|
272
|
-
|
|
273
|
-
const VERDICT = {
|
|
274
|
-
passed: "meaning verified",
|
|
275
|
-
changed: "the meaning may have changed",
|
|
276
|
-
reverted: "rewrite refused; original returned",
|
|
277
|
-
skipped: "meaning not checked",
|
|
278
|
-
unavailable: "the meaning check did not finish",
|
|
279
|
-
};
|
|
280
|
-
|
|
281
|
-
/** Decide a passage once all its calls are back. */
|
|
282
|
-
function settle(unit) {
|
|
283
|
-
const results = unit.calls.map((c) => c.result);
|
|
284
|
-
const failed = results.find((r) => r.error);
|
|
285
|
-
if (failed) return Object.assign(unit, { status: "failed", reason: failed.error });
|
|
286
|
-
const data = results.map((r) => r.data);
|
|
287
|
-
const refined = data.map((d) => String(d.output ?? "").trim())
|
|
288
|
-
.reduce((acc, piece, k) => (k ? acc + (unit.joins[k - 1] ?? " ") + piece : piece), "");
|
|
289
|
-
const verified = data.every((d) => d.meaningCheck === "passed" && d.meaningPreserved === true);
|
|
290
|
-
const notPassed = data.find((d) => d.meaningCheck !== "passed");
|
|
291
|
-
const check = notPassed ? (notPassed.meaningCheck ?? null) : "passed";
|
|
292
|
-
Object.assign(unit, {
|
|
293
|
-
after: refined,
|
|
294
|
-
meaningCheck: check,
|
|
295
|
-
meaningPreserved: data.every((d) => d.meaningPreserved === true) ? true : data.some((d) => d.meaningPreserved === false) ? false : null,
|
|
296
|
-
ids: data.map((d) => d.id).filter(Boolean),
|
|
297
|
-
});
|
|
298
|
-
const unverified = check !== "passed"
|
|
299
|
-
? VERDICT[check] || `meaning check: ${check ?? "missing"}`
|
|
300
|
-
: `meaning not verified (meaningPreserved: ${unit.meaningPreserved})`;
|
|
301
|
-
if (refined.replace(/\s+/g, " ") === unit.text.replace(/\s+/g, " ")) {
|
|
302
|
-
// The engine kept the text. That is a refusal of the meaning check only
|
|
303
|
-
// when it says so; "already fine" and its own quality veto are not.
|
|
304
|
-
const refused = data.find((d) => !d.candidateVetoFired && (d.meaningCheck === "reverted" || d.rejectionReason));
|
|
305
|
-
return Object.assign(unit, refused
|
|
306
|
-
? { status: "kept", reason: refused.rejectionReason || VERDICT.reverted }
|
|
307
|
-
: { status: "unchanged" });
|
|
308
|
-
}
|
|
309
|
-
const mojibake = data.find((d) => d.quality?.encodingArtifacts?.length);
|
|
310
|
-
if (mojibake) return Object.assign(unit, { status: "kept", reason: "encoding artifacts in the output" });
|
|
311
|
-
if (!verified) return Object.assign(unit, { status: "kept", reason: unverified });
|
|
312
|
-
const repaired = repairCodeSpans(unit.text, refined);
|
|
313
|
-
const placed = unit.place(repaired);
|
|
314
|
-
if (!placed.ok) return Object.assign(unit, { status: "kept", reason: placed.reason });
|
|
315
|
-
return Object.assign(unit, { status: "changed", value: placed.value, after: repaired });
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
function passageSummary(unit, n) {
|
|
319
|
-
return {
|
|
320
|
-
n,
|
|
321
|
-
status: unit.status || "pending",
|
|
322
|
-
before: unit.text,
|
|
323
|
-
...(unit.after !== undefined ? { after: unit.after } : {}),
|
|
324
|
-
...(unit.meaningCheck !== undefined ? { meaningCheck: unit.meaningCheck, meaningPreserved: unit.meaningPreserved } : {}),
|
|
325
|
-
...(unit.reason ? { reason: unit.reason } : {}),
|
|
326
|
-
...(unit.ids?.length ? { ids: unit.ids } : {}),
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
/**
|
|
331
|
-
* The whole command. Returns an exit code:
|
|
332
|
-
* 0 done · 2 done, some passages kept their original · 4 out of credit · 1 error
|
|
333
|
-
*/
|
|
334
|
-
export async function runBatch(opts, io = {}) {
|
|
335
|
-
const log = io.log || ((m) => process.stderr.write(`${m}\n`));
|
|
336
|
-
const print = io.print || ((m) => process.stdout.write(`${m}\n`));
|
|
337
|
-
const wait = io.sleep || sleep;
|
|
338
|
-
const now = io.now || Date.now;
|
|
339
|
-
const cwd = io.cwd || process.cwd();
|
|
340
|
-
const fetchImpl = io.fetch || globalThis.fetch;
|
|
341
|
-
const isUrl = /^https?:\/\//i.test(opts.target);
|
|
342
|
-
|
|
343
|
-
if (opts.fast) {
|
|
344
|
-
log("--fast skips the meaning check, and a folder or site run only writes verified changes. Leave --fast out.");
|
|
345
|
-
return EXIT_ERROR;
|
|
346
|
-
}
|
|
347
|
-
if (opts.review) {
|
|
348
|
-
log("--review is for one text. A folder or site run lists every change in its report: run it with --dry-run first, then --yes.");
|
|
349
|
-
return EXIT_ERROR;
|
|
350
|
-
}
|
|
351
|
-
const concurrency = opts.concurrency ?? 2;
|
|
352
|
-
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 4) { log("--concurrency must be 1, 2, 3 or 4."); return EXIT_ERROR; }
|
|
353
|
-
if (opts.maxChars !== undefined && !(Number.isInteger(opts.maxChars) && opts.maxChars > 0)) { log("--max-chars must be a positive whole number."); return EXIT_ERROR; }
|
|
354
|
-
const maxPages = opts.maxPages ?? 20;
|
|
355
|
-
if (!Number.isInteger(maxPages) || maxPages < 1) { log("--max-pages must be a positive whole number."); return EXIT_ERROR; }
|
|
356
|
-
if (opts.site && !isUrl) { log("--site needs a URL, such as https://example.com"); return EXIT_ERROR; }
|
|
357
|
-
// Checked here, not by the API: a bad value would otherwise fail every call.
|
|
358
|
-
if (!DOMAINS.includes(opts.domain ?? "general")) { log(`--domain must be one of: ${DOMAINS.join(", ")}`); return EXIT_ERROR; }
|
|
359
|
-
if (!REGISTERS.includes(opts.register ?? "neutral")) { log(`--register must be one of: ${REGISTERS.join(", ")}`); return EXIT_ERROR; }
|
|
360
|
-
// Before reading anything, so a site is not crawled for nothing. --dry-run
|
|
361
|
-
// sends nothing and needs no key.
|
|
362
|
-
if (!opts.dryRun && !opts.key) { log("Not logged in. Run once: npx altalandi login (or set ALTALANDI_API_KEY)"); return EXIT_ERROR; }
|
|
363
|
-
|
|
364
|
-
let plan;
|
|
365
|
-
try {
|
|
366
|
-
plan = isUrl ? await planWeb({ ...opts, maxPages }, { ...io, fetch: fetchImpl, cwd }, log) : await planFolder(opts, { cwd });
|
|
367
|
-
} catch (e) {
|
|
368
|
-
log(e.usage ? e.message : `Could not read ${opts.target}: ${e.message}`);
|
|
369
|
-
return EXIT_ERROR;
|
|
370
|
-
}
|
|
371
|
-
const calls = buildCalls(plan.files, opts.maxChars);
|
|
372
|
-
const baseUrl = (opts.baseUrl || "https://altalandi.is/api").replace(/\/+$/, "");
|
|
373
|
-
const where = `${plan.label} → ${display(plan.output, cwd)}`;
|
|
374
|
-
const refining = () => plan.files.filter((f) => f.status === "refine");
|
|
375
|
-
|
|
376
|
-
if (opts.maxChars && !refining().length && plan.files.some((f) => f.status === "over-budget")) {
|
|
377
|
-
log(`Nothing fits in --max-chars ${fmt(opts.maxChars)}: ${plan.files.find((f) => f.status === "over-budget").path} alone is more.`);
|
|
378
|
-
return EXIT_ERROR;
|
|
379
|
-
}
|
|
380
|
-
// The public tariff: no key, no text sent, nothing charged.
|
|
381
|
-
const pricing = calls.length ? await fetchPricing(baseUrl, fetchImpl) : null;
|
|
382
|
-
const estimate = estimateLines(plan, calls, pricing, opts);
|
|
383
|
-
|
|
384
|
-
if (opts.dryRun) {
|
|
385
|
-
if (opts.json) {
|
|
386
|
-
print(JSON.stringify({
|
|
387
|
-
dryRun: true, input: plan.label, output: plan.output, estimate,
|
|
388
|
-
files: plan.files.map((f) => ({
|
|
389
|
-
path: f.path, source: f.source, status: f.status, ...(f.note ? { note: f.note } : {}),
|
|
390
|
-
passages: (f.units || []).map((u) => u.text),
|
|
391
|
-
})),
|
|
392
|
-
}, null, 2));
|
|
393
|
-
return EXIT_OK;
|
|
394
|
-
}
|
|
395
|
-
print(`Dry run: nothing is sent and nothing is written.\n${where}\n`);
|
|
396
|
-
for (const f of plan.files) {
|
|
397
|
-
if (f.status === "copy") { print(`${f.path}: copied unchanged (${f.note})`); continue; }
|
|
398
|
-
const chars = f.units.reduce((n, u) => n + u.text.length, 0);
|
|
399
|
-
print(`${f.path}: ${plural(f.units.length, "passage", "passages")}, ${fmt(chars)} characters${f.status === "over-budget" ? " (over --max-chars, not sent)" : ""}`);
|
|
400
|
-
f.units.forEach((u, k) => print(` ${k + 1}. ${preview(u.text)}`));
|
|
401
|
-
}
|
|
402
|
-
print(`\n${estimate.join("\n")}`);
|
|
403
|
-
return EXIT_OK;
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
if (!refining().length) {
|
|
407
|
-
log(`${where}\nNothing to refine: no passages in ${plural(plan.files.length, "file", "files")}.`);
|
|
408
|
-
} else {
|
|
409
|
-
log(`${where}\n${estimate.join("\n")}`);
|
|
410
|
-
if (!opts.yes) {
|
|
411
|
-
const interactive = io.interactive ?? Boolean(process.stdin.isTTY && process.stderr.isTTY);
|
|
412
|
-
if (!interactive) {
|
|
413
|
-
log("Nothing was sent. Without a terminal to ask in, pass --yes to confirm (see the estimate above, or run with --dry-run first).");
|
|
414
|
-
return EXIT_ERROR;
|
|
415
|
-
}
|
|
416
|
-
const confirm = io.confirm || (async (q) => (await import("./update.mjs")).ask(q));
|
|
417
|
-
const n = refining().reduce((k, f) => k + f.units.length, 0);
|
|
418
|
-
if (!(await confirm(`Send ${plural(n, "passage", "passages")} to Altalandi?`))) {
|
|
419
|
-
log("Nothing was sent.");
|
|
420
|
-
return EXIT_ERROR;
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
// Never write inside the input, whatever the paths say.
|
|
426
|
-
const write = async (target, data) => {
|
|
427
|
-
if (plan.input && (samePath(target, plan.input) || isInside(target, plan.input))) {
|
|
428
|
-
throw new Error(`refusing to write inside the input: ${target}`);
|
|
429
|
-
}
|
|
430
|
-
if (plan.outputIsFolder ? !isInside(target, plan.output) : !samePath(target, plan.output)) {
|
|
431
|
-
throw new Error(`refusing to write outside the output: ${target}`);
|
|
432
|
-
}
|
|
433
|
-
await mkdir(path.dirname(target), { recursive: true });
|
|
434
|
-
await writeFile(target, data);
|
|
435
|
-
};
|
|
436
|
-
|
|
437
|
-
const state = { stop: null, pausedUntil: 0, errorsInARow: 0 };
|
|
438
|
-
const stop = (reason, message) => { if (!state.stop) state.stop = { reason, message }; };
|
|
439
|
-
|
|
440
|
-
for (const f of plan.files.filter((x) => x.status === "copy")) {
|
|
441
|
-
try { await write(f.target, f.bytes ?? f.text); } catch (e) { stop("error", `Could not write ${f.target}: ${e.message}`); }
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
const requestOptions = {
|
|
445
|
-
apiKey: opts.key, baseUrl, domain: opts.domain, register: opts.register,
|
|
446
|
-
// Only verified changes are written, so ask for the check explicitly.
|
|
447
|
-
preserveMeaning: true,
|
|
448
|
-
...(opts.noStore ? { contribute: false } : {}),
|
|
449
|
-
...(io.fetch ? { fetch: io.fetch } : {}),
|
|
450
|
-
};
|
|
451
|
-
|
|
452
|
-
const pause = async () => {
|
|
453
|
-
for (let left = state.pausedUntil - now(); left > 0; left = state.pausedUntil - now()) await wait(left);
|
|
454
|
-
};
|
|
455
|
-
|
|
456
|
-
const callOnce = async (call) => {
|
|
457
|
-
let rateLimited = 0;
|
|
458
|
-
let transient = 0;
|
|
459
|
-
for (;;) {
|
|
460
|
-
await pause();
|
|
461
|
-
if (state.stop) return { skipped: true };
|
|
462
|
-
try {
|
|
463
|
-
return { data: await refine(call.text, requestOptions) };
|
|
464
|
-
} catch (e) {
|
|
465
|
-
if (e.status === 429) {
|
|
466
|
-
rateLimited++;
|
|
467
|
-
const ms = Number.isFinite(e.retryAfterMs) ? e.retryAfterMs : Math.min(60_000, 2000 * 2 ** (rateLimited - 1));
|
|
468
|
-
if (ms > MAX_RETRY_WAIT_MS) {
|
|
469
|
-
stop("credit", `${e.message} Try again in about ${Math.ceil(ms / 60_000)} min.`);
|
|
470
|
-
return { skipped: true };
|
|
471
|
-
}
|
|
472
|
-
if (rateLimited > MAX_RATE_LIMIT_RETRIES) {
|
|
473
|
-
stop("error", `Still rate limited after ${MAX_RATE_LIMIT_RETRIES} retries: ${e.message}`);
|
|
474
|
-
return { skipped: true };
|
|
475
|
-
}
|
|
476
|
-
if (now() + ms > state.pausedUntil) {
|
|
477
|
-
state.pausedUntil = now() + ms;
|
|
478
|
-
log(`Rate limited (429): waiting ${Math.ceil(ms / 1000)} s, as the server asked.`);
|
|
479
|
-
}
|
|
480
|
-
continue;
|
|
481
|
-
}
|
|
482
|
-
if (e.status === 402) { stop("credit", e.message); return { skipped: true }; }
|
|
483
|
-
if (e.status === 401 || e.status === 403) { stop("error", `${e.message} → The key is invalid or revoked. Run: npx altalandi login`); return { skipped: true }; }
|
|
484
|
-
// Network errors, timeouts and server errors get two more tries.
|
|
485
|
-
if ((!e.status || e.status === 408 || e.status >= 500) && transient < MAX_TRANSIENT_RETRIES) {
|
|
486
|
-
transient++;
|
|
487
|
-
await wait(1000 * 2 ** transient);
|
|
488
|
-
continue;
|
|
489
|
-
}
|
|
490
|
-
return { error: e.message };
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
};
|
|
494
|
-
|
|
495
|
-
const progress = (file) => {
|
|
496
|
-
const u = file.units;
|
|
497
|
-
const changed = u.filter((x) => x.status === "changed").length;
|
|
498
|
-
const kept = u.filter((x) => x.status === "kept").length;
|
|
499
|
-
const failed = u.filter((x) => x.status === "failed").length;
|
|
500
|
-
const extra = [kept && `${kept} kept as they were (check did not pass)`, failed && `${failed} failed`].filter(Boolean).join(", ");
|
|
501
|
-
log(`${kept || failed ? "!" : "✓"} ${file.path}: ${changed} of ${plural(u.length, "passage", "passages")} changed${extra ? `; ${extra}` : ""}`);
|
|
502
|
-
};
|
|
503
|
-
|
|
504
|
-
const finish = async (file) => {
|
|
505
|
-
file.status = "writing"; // before the first await, so no other worker writes it too
|
|
506
|
-
const values = new Map(file.units.filter((u) => u.status === "changed").map((u) => [u, u.value]));
|
|
507
|
-
try {
|
|
508
|
-
await write(file.target, values.size ? assemble(file.text, file.units, values) : (file.bytes ?? file.text));
|
|
509
|
-
file.status = values.size ? "written" : "unchanged";
|
|
510
|
-
progress(file);
|
|
511
|
-
} catch (e) {
|
|
512
|
-
file.status = "failed";
|
|
513
|
-
file.note = e.message;
|
|
514
|
-
stop("error", `Could not write ${file.target}: ${e.message}`);
|
|
515
|
-
}
|
|
516
|
-
};
|
|
517
|
-
|
|
518
|
-
const done = async (call) => {
|
|
519
|
-
if (call.result.skipped) return;
|
|
520
|
-
if (call.result.error) {
|
|
521
|
-
state.errorsInARow++;
|
|
522
|
-
if (state.errorsInARow >= ERRORS_IN_A_ROW_TO_STOP) stop("error", `${state.errorsInARow} calls in a row failed; the last: ${call.result.error}`);
|
|
523
|
-
} else {
|
|
524
|
-
state.errorsInARow = 0;
|
|
525
|
-
}
|
|
526
|
-
for (const unit of call.units) {
|
|
527
|
-
if (unit.status || !unit.calls.every((c) => c.result && !c.result.skipped)) continue;
|
|
528
|
-
settle(unit);
|
|
529
|
-
const file = unit.file;
|
|
530
|
-
if (file.status === "refine" && file.units.every((u) => u.status)) await finish(file);
|
|
531
|
-
}
|
|
532
|
-
};
|
|
533
|
-
|
|
534
|
-
const hadWork = refining().length > 0;
|
|
535
|
-
const queue = [...calls];
|
|
536
|
-
const worker = async () => {
|
|
537
|
-
while (queue.length && !state.stop) {
|
|
538
|
-
const call = queue.shift();
|
|
539
|
-
call.result = await callOnce(call);
|
|
540
|
-
await done(call);
|
|
541
|
-
}
|
|
542
|
-
};
|
|
543
|
-
await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(1, calls.length)) }, worker));
|
|
544
|
-
|
|
545
|
-
for (const f of plan.files) if (f.status === "refine") f.status = "unfinished";
|
|
546
|
-
|
|
547
|
-
// ── results ───────────────────────────────────────────────────────────────
|
|
548
|
-
const files = plan.files.map((f) => ({
|
|
549
|
-
path: f.path,
|
|
550
|
-
...(f.source && isUrl ? { source: f.source } : {}),
|
|
551
|
-
output: display(f.target, cwd),
|
|
552
|
-
status: f.status,
|
|
553
|
-
...(f.note ? { note: f.note } : {}),
|
|
554
|
-
passages: (f.units || []).map((u, k) => passageSummary(u, k + 1)),
|
|
555
|
-
}));
|
|
556
|
-
const passages = files.flatMap((f) => f.passages);
|
|
557
|
-
const count = (s) => passages.filter((p) => p.status === s).length;
|
|
558
|
-
const summary = {
|
|
559
|
-
input: plan.label,
|
|
560
|
-
output: display(plan.output, cwd),
|
|
561
|
-
mode: plan.mode,
|
|
562
|
-
domain: opts.domain,
|
|
563
|
-
register: opts.register,
|
|
564
|
-
finishedAt: new Date(now()).toISOString(),
|
|
565
|
-
stopped: state.stop,
|
|
566
|
-
totals: {
|
|
567
|
-
files: files.length,
|
|
568
|
-
written: files.filter((f) => f.status === "written").length,
|
|
569
|
-
unchanged: files.filter((f) => f.status === "unchanged").length,
|
|
570
|
-
copied: files.filter((f) => f.status === "copy").length,
|
|
571
|
-
unfinished: files.filter((f) => f.status === "unfinished").length,
|
|
572
|
-
overBudget: files.filter((f) => f.status === "over-budget").length,
|
|
573
|
-
passages: passages.length,
|
|
574
|
-
changed: count("changed"),
|
|
575
|
-
unchangedPassages: count("unchanged"),
|
|
576
|
-
kept: count("kept"),
|
|
577
|
-
failed: count("failed"),
|
|
578
|
-
characters: calls.filter((c) => c.result && !c.result.skipped).reduce((n, c) => n + c.text.length, 0),
|
|
579
|
-
},
|
|
580
|
-
...(plan.skipped?.length ? { skippedPages: plan.skipped } : {}),
|
|
581
|
-
files,
|
|
582
|
-
};
|
|
583
|
-
|
|
584
|
-
let exit = EXIT_OK;
|
|
585
|
-
if (summary.totals.kept) exit = EXIT_KEPT;
|
|
586
|
-
if (summary.totals.failed || summary.totals.overBudget || state.stop) exit = EXIT_ERROR;
|
|
587
|
-
if (state.stop?.reason === "credit") exit = EXIT_CREDIT;
|
|
588
|
-
summary.exitCode = exit;
|
|
589
|
-
|
|
590
|
-
if (passages.length) {
|
|
591
|
-
// Next to the output by default (./altalandi-reports for `altalandi ./docs`),
|
|
592
|
-
// and never inside the input, where the next run would pick it up.
|
|
593
|
-
const beside = path.join(path.dirname(plan.output), "altalandi-reports");
|
|
594
|
-
let reportDir = path.resolve(cwd, opts.reportDir || process.env.ALTALANDI_REPORT_DIR || beside);
|
|
595
|
-
if (plan.input && (samePath(reportDir, plan.input) || isInside(reportDir, plan.input))) reportDir = beside;
|
|
596
|
-
try {
|
|
597
|
-
summary.report = await writeBatchReport(summary, reportDir);
|
|
598
|
-
} catch (e) {
|
|
599
|
-
log(`Could not write the report: ${e.message}`);
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
const t = summary.totals;
|
|
604
|
-
if (state.stop) {
|
|
605
|
-
log(`\nStopped: ${state.stop.message}`);
|
|
606
|
-
if (state.stop.reason === "credit") log("→ Add credit at https://altalandi.is/
|
|
607
|
-
const finished = files.filter((f) => ["written", "unchanged"].includes(f.status)).map((f) => f.path);
|
|
608
|
-
const unfinished = files.filter((f) => f.status === "unfinished").map((f) => f.path);
|
|
609
|
-
log(`Finished (${finished.length}): ${finished.join(", ") || "none"}`);
|
|
610
|
-
log(`Not finished, not written (${unfinished.length}): ${unfinished.join(", ") || "none"}`);
|
|
611
|
-
}
|
|
612
|
-
log(`\n${state.stop ? "Partly done" : "Done"}: ${plural(t.written + t.unchanged + t.copied, "file", "files")} in ${display(plan.output, cwd)}`
|
|
613
|
-
+ `${t.copied ? ` (${fmt(t.copied)} copied unchanged)` : ""}.`
|
|
614
|
-
+ (hadWork ? ` Passages: ${fmt(t.changed)} changed, ${fmt(t.unchangedPassages)} needed no change, ${fmt(t.kept)} kept as they were, ${fmt(t.failed)} failed.` : ""));
|
|
615
|
-
if (t.kept) log(`${plural(t.kept, "passage keeps", "passages keep")} the original text because a check did not pass; the report lists ${t.kept === 1 ? "it" : "them"}.`);
|
|
616
|
-
if (t.overBudget) log(`${plural(t.overBudget, "file was", "files were")} left out for --max-chars.`);
|
|
617
|
-
if (summary.report) log(`Report: ${display(summary.report, cwd)}`);
|
|
618
|
-
if (opts.json) print(JSON.stringify(summary, null, 2));
|
|
619
|
-
return exit;
|
|
620
|
-
}
|
|
1
|
+
// `altalandi <folder | url>`: refine a whole folder of documents, one web page
|
|
2
|
+
// or a site in one command, and write the Icelandic next to the original.
|
|
3
|
+
//
|
|
4
|
+
// The order of things is the safety story:
|
|
5
|
+
// 1. plan read every document and cut it into passages (segment.mjs,
|
|
6
|
+
// web.mjs); nothing is sent yet
|
|
7
|
+
// 2. estimate files, passages, characters and billing units, printed before
|
|
8
|
+
// anything is sent
|
|
9
|
+
// 3. consent a y/N question in a terminal, or --yes; with neither, nothing
|
|
10
|
+
// is sent (--dry-run stops here on purpose)
|
|
11
|
+
// 4. refine one call per passage, 2 at a time by default; a 429 waits for
|
|
12
|
+
// Retry-After, a 402 stops the run cleanly
|
|
13
|
+
// 5. write each file once all its passages are back; a passage whose
|
|
14
|
+
// meaning check did not pass keeps its original text; nothing is
|
|
15
|
+
// ever written inside the input
|
|
16
|
+
// 6. report one Markdown summary of every file and every change
|
|
17
|
+
//
|
|
18
|
+
// Every side effect is injectable (io), so the tests need no network, terminal
|
|
19
|
+
// or API key.
|
|
20
|
+
|
|
21
|
+
import { existsSync, statSync } from "node:fs";
|
|
22
|
+
import { mkdir, readdir, readFile, realpath, writeFile } from "node:fs/promises";
|
|
23
|
+
import path from "node:path";
|
|
24
|
+
|
|
25
|
+
import { refine } from "./client.mjs";
|
|
26
|
+
import { writeBatchReport } from "./report.mjs";
|
|
27
|
+
import { assemble, FORMATS, isConfigJson, repairCodeSpans, segment, segmentMarkdown, splitForApi } from "./segment.mjs";
|
|
28
|
+
import { pagePath, readPage, readSite, safeSegment, userAgent } from "./web.mjs";
|
|
29
|
+
|
|
30
|
+
export const EXIT_OK = 0;
|
|
31
|
+
export const EXIT_ERROR = 1;
|
|
32
|
+
export const EXIT_KEPT = 2; // finished, but some passages kept their original text
|
|
33
|
+
export const EXIT_CREDIT = 4; // stopped: out of credit or allowance (as for a single text)
|
|
34
|
+
|
|
35
|
+
// altalandi-reports: the reports of earlier runs are not documents to refine.
|
|
36
|
+
const SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", "altalandi-reports"]);
|
|
37
|
+
const MAX_FILE_BYTES = 5 * 1024 * 1024;
|
|
38
|
+
const MAX_RETRY_WAIT_MS = 5 * 60_000; // a longer Retry-After means "come back later"
|
|
39
|
+
const MAX_RATE_LIMIT_RETRIES = 8;
|
|
40
|
+
const MAX_TRANSIENT_RETRIES = 2;
|
|
41
|
+
const ERRORS_IN_A_ROW_TO_STOP = 3;
|
|
42
|
+
const DOMAINS = ["general", "marketing", "data-analysis", "childrens-fiction"];
|
|
43
|
+
const REGISTERS = ["formal", "neutral", "warm"];
|
|
44
|
+
|
|
45
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
46
|
+
const fmt = (n) => Math.round(n).toLocaleString("en-US");
|
|
47
|
+
const plural = (n, one, many) => `${fmt(n)} ${n === 1 ? one : many}`;
|
|
48
|
+
const fold = (p) => (process.platform === "win32" || process.platform === "darwin" ? p.toLowerCase() : p);
|
|
49
|
+
const samePath = (a, b) => fold(path.resolve(a)) === fold(path.resolve(b));
|
|
50
|
+
const isInside = (child, parent) => {
|
|
51
|
+
const rel = path.relative(fold(path.resolve(parent)), fold(path.resolve(child)));
|
|
52
|
+
return Boolean(rel) && !rel.startsWith("..") && !path.isAbsolute(rel);
|
|
53
|
+
};
|
|
54
|
+
const display = (p, cwd) => {
|
|
55
|
+
const rel = path.relative(cwd, p);
|
|
56
|
+
return rel && !rel.startsWith("..") && !path.isAbsolute(rel) ? rel : p;
|
|
57
|
+
};
|
|
58
|
+
const preview = (s, n = 72) => {
|
|
59
|
+
const one = String(s).replace(/\s+/g, " ").trim();
|
|
60
|
+
return one.length > n ? `${one.slice(0, n - 1)}…` : one;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
/** Every whitespace token is at least one billing unit; a long unbroken token
|
|
64
|
+
* is one unit per 20 characters (the server's rule, docs/api/README.md). */
|
|
65
|
+
const billingTokens = (text) => text.split(/\s+/).filter(Boolean)
|
|
66
|
+
.reduce((n, t) => n + Math.max(1, Math.ceil(t.length / 20)), 0);
|
|
67
|
+
|
|
68
|
+
/** The real path of p, or of its deepest existing ancestor plus the rest, so a
|
|
69
|
+
* symlinked --out is compared where it really points. */
|
|
70
|
+
async function realish(p) {
|
|
71
|
+
const abs = path.resolve(p);
|
|
72
|
+
const rest = [];
|
|
73
|
+
let probe = abs;
|
|
74
|
+
for (;;) {
|
|
75
|
+
try {
|
|
76
|
+
return path.join(await realpath(probe), ...rest.reverse());
|
|
77
|
+
} catch {
|
|
78
|
+
const parent = path.dirname(probe);
|
|
79
|
+
if (parent === probe) return abs;
|
|
80
|
+
rest.push(path.basename(probe));
|
|
81
|
+
probe = parent;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function walk(root) {
|
|
87
|
+
const files = [];
|
|
88
|
+
const visit = async (dir, rel) => {
|
|
89
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
90
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
91
|
+
for (const e of entries) {
|
|
92
|
+
// Hidden files and folders, and symlinks (no loops, no escaping the input).
|
|
93
|
+
if (e.name.startsWith(".") || e.isSymbolicLink()) continue;
|
|
94
|
+
const childRel = rel ? `${rel}/${e.name}` : e.name;
|
|
95
|
+
if (e.isDirectory()) {
|
|
96
|
+
if (!SKIP_DIRS.has(e.name)) await visit(path.join(dir, e.name), childRel);
|
|
97
|
+
} else if (e.isFile() && FORMATS[path.extname(e.name).toLowerCase()]) {
|
|
98
|
+
files.push(childRel);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
await visit(root, "");
|
|
103
|
+
return files;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function usage(message) {
|
|
107
|
+
const e = new Error(message);
|
|
108
|
+
e.usage = true;
|
|
109
|
+
return e;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// ── plans ───────────────────────────────────────────────────────────────────
|
|
113
|
+
|
|
114
|
+
async function planFolder(opts, io) {
|
|
115
|
+
const cwd = io.cwd || process.cwd();
|
|
116
|
+
const given = path.resolve(cwd, opts.target);
|
|
117
|
+
let input;
|
|
118
|
+
try { input = await realpath(given); } catch { throw usage(`No such folder: ${opts.target}`); }
|
|
119
|
+
if (!statSync(input).isDirectory()) throw usage(`${opts.target} is not a folder.`);
|
|
120
|
+
if (!path.basename(input)) throw usage("Give the output folder with --out.");
|
|
121
|
+
const output = path.resolve(cwd, opts.out || path.join(path.dirname(given), `${path.basename(given)}-is`));
|
|
122
|
+
const outputReal = await realish(output);
|
|
123
|
+
if (samePath(outputReal, input) || isInside(outputReal, input) || isInside(input, outputReal)) {
|
|
124
|
+
throw usage(`--out must be a separate folder, not the input folder, inside it or around it: ${output}`);
|
|
125
|
+
}
|
|
126
|
+
if (existsSync(output) && !statSync(output).isDirectory()) throw usage(`--out ${output} is a file; give a folder.`);
|
|
127
|
+
|
|
128
|
+
const files = [];
|
|
129
|
+
for (const rel of await walk(input)) {
|
|
130
|
+
const source = path.join(input, ...rel.split("/"));
|
|
131
|
+
const file = {
|
|
132
|
+
path: rel, source, target: path.join(output, ...rel.split("/")), format: FORMATS[path.extname(rel).toLowerCase()],
|
|
133
|
+
};
|
|
134
|
+
file.bytes = await readFile(source);
|
|
135
|
+
let text;
|
|
136
|
+
if (file.bytes.length > MAX_FILE_BYTES) file.note = "larger than 5 MB";
|
|
137
|
+
else {
|
|
138
|
+
try { text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(file.bytes); } catch { file.note = "not UTF-8 text"; }
|
|
139
|
+
}
|
|
140
|
+
if (text !== undefined) {
|
|
141
|
+
if (file.format === "json" && isConfigJson(path.basename(rel))) file.note = "configuration, not a string catalog";
|
|
142
|
+
else {
|
|
143
|
+
const seg = segment(text, file.format);
|
|
144
|
+
file.text = text;
|
|
145
|
+
file.units = seg.units;
|
|
146
|
+
file.note = seg.note || (seg.units.length ? undefined : "nothing to refine");
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
file.status = file.note ? "copy" : "refine";
|
|
150
|
+
files.push(file);
|
|
151
|
+
}
|
|
152
|
+
return { mode: "folder", label: opts.target, input, output, outputIsFolder: true, files };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const withSource = (page) => `<!-- Source: ${page.url} -->\n\n${page.markdown}\n`;
|
|
156
|
+
|
|
157
|
+
function pageFile(page, rel, target) {
|
|
158
|
+
const text = withSource(page);
|
|
159
|
+
const { units } = segmentMarkdown(text);
|
|
160
|
+
return {
|
|
161
|
+
path: rel, source: page.url, target, format: "md", text, units,
|
|
162
|
+
status: units.length ? "refine" : "copy", note: units.length ? undefined : "nothing to refine",
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async function planWeb(opts, io, log) {
|
|
167
|
+
const cwd = io.cwd || process.cwd();
|
|
168
|
+
const web = { fetch: io.fetch, sleep: io.sleep, userAgent: userAgent(io.version || "0") };
|
|
169
|
+
if (!opts.site) {
|
|
170
|
+
const page = await readPage(opts.target, web);
|
|
171
|
+
const last = pagePath(page.url).replace(/\.md$/, "").split("/").pop();
|
|
172
|
+
const name = `${last === "index" ? safeSegment(new URL(page.url).host) : last}-is.md`;
|
|
173
|
+
let target = path.resolve(cwd, opts.out || name);
|
|
174
|
+
if (existsSync(target) && statSync(target).isDirectory()) target = path.join(target, name);
|
|
175
|
+
return { mode: "page", label: opts.target, output: target, outputIsFolder: false, files: [pageFile(page, path.basename(target), target)] };
|
|
176
|
+
}
|
|
177
|
+
log(`Reading ${opts.target} (up to ${plural(opts.maxPages, "page", "pages")}) …`);
|
|
178
|
+
const { pages, skipped, source } = await readSite(opts.target, {
|
|
179
|
+
maxPages: opts.maxPages, io: web, onPage: (p) => log(` read ${p.url}`),
|
|
180
|
+
});
|
|
181
|
+
const output = path.resolve(cwd, opts.out || `${safeSegment(new URL(pages[0].url).host)}-is`);
|
|
182
|
+
if (existsSync(output) && !statSync(output).isDirectory()) throw usage(`--out ${output} is a file; --site writes a folder.`);
|
|
183
|
+
const taken = new Set();
|
|
184
|
+
const files = pages.map((page) => {
|
|
185
|
+
let rel = pagePath(page.url);
|
|
186
|
+
for (let k = 2; taken.has(rel.toLowerCase()); k++) rel = pagePath(page.url).replace(/\.md$/, `-${k}.md`);
|
|
187
|
+
taken.add(rel.toLowerCase());
|
|
188
|
+
return pageFile(page, rel, path.join(output, ...rel.split("/")));
|
|
189
|
+
});
|
|
190
|
+
return { mode: "site", label: opts.target, output, outputIsFolder: true, files, skipped, discovery: source };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// ── calls ───────────────────────────────────────────────────────────────────
|
|
194
|
+
|
|
195
|
+
/** One call per distinct piece of text: a passage seen twice is paid once. A
|
|
196
|
+
* passage over the API limit becomes several pieces cut at sentences. */
|
|
197
|
+
function buildCalls(files, maxChars) {
|
|
198
|
+
const calls = new Map();
|
|
199
|
+
let budgetUsed = 0;
|
|
200
|
+
let overBudget = false;
|
|
201
|
+
for (const file of files) {
|
|
202
|
+
if (file.status !== "refine") continue;
|
|
203
|
+
const fresh = new Map();
|
|
204
|
+
for (const unit of file.units) {
|
|
205
|
+
const { pieces, joins } = splitForApi(unit.text);
|
|
206
|
+
unit.joins = joins;
|
|
207
|
+
unit.calls = pieces.map((text) => calls.get(text) || fresh.get(text) || fresh.set(text, { text, units: [] }).get(text));
|
|
208
|
+
}
|
|
209
|
+
const added = [...fresh.values()].reduce((n, c) => n + c.text.length, 0);
|
|
210
|
+
if (overBudget || (maxChars && budgetUsed + added > maxChars)) {
|
|
211
|
+
overBudget = true;
|
|
212
|
+
file.status = "over-budget";
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
budgetUsed += added;
|
|
216
|
+
for (const [text, call] of fresh) calls.set(text, call);
|
|
217
|
+
for (const unit of file.units) {
|
|
218
|
+
unit.file = file;
|
|
219
|
+
for (const call of new Set(unit.calls)) call.units.push(unit);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return [...calls.values()];
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async function fetchPricing(baseUrl, fetchImpl) {
|
|
226
|
+
const controller = new AbortController();
|
|
227
|
+
const timer = setTimeout(() => controller.abort(), 5000);
|
|
228
|
+
try {
|
|
229
|
+
const res = await fetchImpl(`${baseUrl}/v1/pricing`, { headers: { accept: "application/json" }, signal: controller.signal });
|
|
230
|
+
if (!res.ok) return null;
|
|
231
|
+
const p = JSON.parse(await res.text());
|
|
232
|
+
const lane = p?.tiers?.standard?.fast;
|
|
233
|
+
if (!Number.isFinite(p?.minimumBillableUnits) || !lane) return null;
|
|
234
|
+
return {
|
|
235
|
+
minUnits: p.minimumBillableUnits,
|
|
236
|
+
multiplier: Number(lane.multiplier) || 1,
|
|
237
|
+
iskPer1000: Number(lane.iskPer1000Units),
|
|
238
|
+
freeDaily: Number(p.accountFreeDailyUnits) || 0,
|
|
239
|
+
};
|
|
240
|
+
} catch {
|
|
241
|
+
return null;
|
|
242
|
+
} finally {
|
|
243
|
+
clearTimeout(timer);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function estimateLines(plan, calls, pricing, opts) {
|
|
248
|
+
const refining = plan.files.filter((f) => f.status === "refine");
|
|
249
|
+
const passages = refining.reduce((n, f) => n + f.units.length, 0);
|
|
250
|
+
const chars = calls.reduce((n, c) => n + c.text.length, 0);
|
|
251
|
+
const lines = [
|
|
252
|
+
`${plural(refining.length, "file", "files")}, ${plural(passages, "passage", "passages")}, about ${fmt(chars)} characters`
|
|
253
|
+
+ `${calls.length !== passages ? ` (${plural(calls.length, "call", "calls")}: repeated passages are sent once)` : ""}.`,
|
|
254
|
+
];
|
|
255
|
+
const copied = plan.files.filter((f) => f.status === "copy").length;
|
|
256
|
+
if (copied) lines.push(`${plural(copied, "more file has", "more files have")} nothing to refine and ${copied === 1 ? "is" : "are"} copied unchanged.`);
|
|
257
|
+
const over = plan.files.filter((f) => f.status === "over-budget");
|
|
258
|
+
if (over.length) {
|
|
259
|
+
lines.push(`--max-chars ${fmt(opts.maxChars)}: ${plural(over.length, "file does", "files do")} not fit and will not be sent `
|
|
260
|
+
+ `(from ${over[0].path} on).`);
|
|
261
|
+
}
|
|
262
|
+
if (pricing && calls.length) {
|
|
263
|
+
const units = calls.reduce((n, c) => n + Math.max(pricing.minUnits, billingTokens(c.text)), 0) * pricing.multiplier;
|
|
264
|
+
const isk = Number.isFinite(pricing.iskPer1000) ? `, about ${fmt((units * pricing.iskPer1000) / 1000)} kr if paid from credit` : "";
|
|
265
|
+
lines.push(`Billing: about ${fmt(units)} units (at least ${fmt(pricing.minUnits)} a call)${isk}.`
|
|
266
|
+
+ `${pricing.freeDaily ? ` The free allowance is ${fmt(pricing.freeDaily)} units a day.` : ""}`);
|
|
267
|
+
}
|
|
268
|
+
return lines;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ── the run ─────────────────────────────────────────────────────────────────
|
|
272
|
+
|
|
273
|
+
const VERDICT = {
|
|
274
|
+
passed: "meaning verified",
|
|
275
|
+
changed: "the meaning may have changed",
|
|
276
|
+
reverted: "rewrite refused; original returned",
|
|
277
|
+
skipped: "meaning not checked",
|
|
278
|
+
unavailable: "the meaning check did not finish",
|
|
279
|
+
};
|
|
280
|
+
|
|
281
|
+
/** Decide a passage once all its calls are back. */
|
|
282
|
+
function settle(unit) {
|
|
283
|
+
const results = unit.calls.map((c) => c.result);
|
|
284
|
+
const failed = results.find((r) => r.error);
|
|
285
|
+
if (failed) return Object.assign(unit, { status: "failed", reason: failed.error });
|
|
286
|
+
const data = results.map((r) => r.data);
|
|
287
|
+
const refined = data.map((d) => String(d.output ?? "").trim())
|
|
288
|
+
.reduce((acc, piece, k) => (k ? acc + (unit.joins[k - 1] ?? " ") + piece : piece), "");
|
|
289
|
+
const verified = data.every((d) => d.meaningCheck === "passed" && d.meaningPreserved === true);
|
|
290
|
+
const notPassed = data.find((d) => d.meaningCheck !== "passed");
|
|
291
|
+
const check = notPassed ? (notPassed.meaningCheck ?? null) : "passed";
|
|
292
|
+
Object.assign(unit, {
|
|
293
|
+
after: refined,
|
|
294
|
+
meaningCheck: check,
|
|
295
|
+
meaningPreserved: data.every((d) => d.meaningPreserved === true) ? true : data.some((d) => d.meaningPreserved === false) ? false : null,
|
|
296
|
+
ids: data.map((d) => d.id).filter(Boolean),
|
|
297
|
+
});
|
|
298
|
+
const unverified = check !== "passed"
|
|
299
|
+
? VERDICT[check] || `meaning check: ${check ?? "missing"}`
|
|
300
|
+
: `meaning not verified (meaningPreserved: ${unit.meaningPreserved})`;
|
|
301
|
+
if (refined.replace(/\s+/g, " ") === unit.text.replace(/\s+/g, " ")) {
|
|
302
|
+
// The engine kept the text. That is a refusal of the meaning check only
|
|
303
|
+
// when it says so; "already fine" and its own quality veto are not.
|
|
304
|
+
const refused = data.find((d) => !d.candidateVetoFired && (d.meaningCheck === "reverted" || d.rejectionReason));
|
|
305
|
+
return Object.assign(unit, refused
|
|
306
|
+
? { status: "kept", reason: refused.rejectionReason || VERDICT.reverted }
|
|
307
|
+
: { status: "unchanged" });
|
|
308
|
+
}
|
|
309
|
+
const mojibake = data.find((d) => d.quality?.encodingArtifacts?.length);
|
|
310
|
+
if (mojibake) return Object.assign(unit, { status: "kept", reason: "encoding artifacts in the output" });
|
|
311
|
+
if (!verified) return Object.assign(unit, { status: "kept", reason: unverified });
|
|
312
|
+
const repaired = repairCodeSpans(unit.text, refined);
|
|
313
|
+
const placed = unit.place(repaired);
|
|
314
|
+
if (!placed.ok) return Object.assign(unit, { status: "kept", reason: placed.reason });
|
|
315
|
+
return Object.assign(unit, { status: "changed", value: placed.value, after: repaired });
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function passageSummary(unit, n) {
|
|
319
|
+
return {
|
|
320
|
+
n,
|
|
321
|
+
status: unit.status || "pending",
|
|
322
|
+
before: unit.text,
|
|
323
|
+
...(unit.after !== undefined ? { after: unit.after } : {}),
|
|
324
|
+
...(unit.meaningCheck !== undefined ? { meaningCheck: unit.meaningCheck, meaningPreserved: unit.meaningPreserved } : {}),
|
|
325
|
+
...(unit.reason ? { reason: unit.reason } : {}),
|
|
326
|
+
...(unit.ids?.length ? { ids: unit.ids } : {}),
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* The whole command. Returns an exit code:
|
|
332
|
+
* 0 done · 2 done, some passages kept their original · 4 out of credit · 1 error
|
|
333
|
+
*/
|
|
334
|
+
export async function runBatch(opts, io = {}) {
|
|
335
|
+
const log = io.log || ((m) => process.stderr.write(`${m}\n`));
|
|
336
|
+
const print = io.print || ((m) => process.stdout.write(`${m}\n`));
|
|
337
|
+
const wait = io.sleep || sleep;
|
|
338
|
+
const now = io.now || Date.now;
|
|
339
|
+
const cwd = io.cwd || process.cwd();
|
|
340
|
+
const fetchImpl = io.fetch || globalThis.fetch;
|
|
341
|
+
const isUrl = /^https?:\/\//i.test(opts.target);
|
|
342
|
+
|
|
343
|
+
if (opts.fast) {
|
|
344
|
+
log("--fast skips the meaning check, and a folder or site run only writes verified changes. Leave --fast out.");
|
|
345
|
+
return EXIT_ERROR;
|
|
346
|
+
}
|
|
347
|
+
if (opts.review) {
|
|
348
|
+
log("--review is for one text. A folder or site run lists every change in its report: run it with --dry-run first, then --yes.");
|
|
349
|
+
return EXIT_ERROR;
|
|
350
|
+
}
|
|
351
|
+
const concurrency = opts.concurrency ?? 2;
|
|
352
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 4) { log("--concurrency must be 1, 2, 3 or 4."); return EXIT_ERROR; }
|
|
353
|
+
if (opts.maxChars !== undefined && !(Number.isInteger(opts.maxChars) && opts.maxChars > 0)) { log("--max-chars must be a positive whole number."); return EXIT_ERROR; }
|
|
354
|
+
const maxPages = opts.maxPages ?? 20;
|
|
355
|
+
if (!Number.isInteger(maxPages) || maxPages < 1) { log("--max-pages must be a positive whole number."); return EXIT_ERROR; }
|
|
356
|
+
if (opts.site && !isUrl) { log("--site needs a URL, such as https://example.com"); return EXIT_ERROR; }
|
|
357
|
+
// Checked here, not by the API: a bad value would otherwise fail every call.
|
|
358
|
+
if (!DOMAINS.includes(opts.domain ?? "general")) { log(`--domain must be one of: ${DOMAINS.join(", ")}`); return EXIT_ERROR; }
|
|
359
|
+
if (!REGISTERS.includes(opts.register ?? "neutral")) { log(`--register must be one of: ${REGISTERS.join(", ")}`); return EXIT_ERROR; }
|
|
360
|
+
// Before reading anything, so a site is not crawled for nothing. --dry-run
|
|
361
|
+
// sends nothing and needs no key.
|
|
362
|
+
if (!opts.dryRun && !opts.key) { log("Not logged in. Run once: npx altalandi login (or set ALTALANDI_API_KEY)"); return EXIT_ERROR; }
|
|
363
|
+
|
|
364
|
+
let plan;
|
|
365
|
+
try {
|
|
366
|
+
plan = isUrl ? await planWeb({ ...opts, maxPages }, { ...io, fetch: fetchImpl, cwd }, log) : await planFolder(opts, { cwd });
|
|
367
|
+
} catch (e) {
|
|
368
|
+
log(e.usage ? e.message : `Could not read ${opts.target}: ${e.message}`);
|
|
369
|
+
return EXIT_ERROR;
|
|
370
|
+
}
|
|
371
|
+
const calls = buildCalls(plan.files, opts.maxChars);
|
|
372
|
+
const baseUrl = (opts.baseUrl || "https://altalandi.is/api").replace(/\/+$/, "");
|
|
373
|
+
const where = `${plan.label} → ${display(plan.output, cwd)}`;
|
|
374
|
+
const refining = () => plan.files.filter((f) => f.status === "refine");
|
|
375
|
+
|
|
376
|
+
if (opts.maxChars && !refining().length && plan.files.some((f) => f.status === "over-budget")) {
|
|
377
|
+
log(`Nothing fits in --max-chars ${fmt(opts.maxChars)}: ${plan.files.find((f) => f.status === "over-budget").path} alone is more.`);
|
|
378
|
+
return EXIT_ERROR;
|
|
379
|
+
}
|
|
380
|
+
// The public tariff: no key, no text sent, nothing charged.
|
|
381
|
+
const pricing = calls.length ? await fetchPricing(baseUrl, fetchImpl) : null;
|
|
382
|
+
const estimate = estimateLines(plan, calls, pricing, opts);
|
|
383
|
+
|
|
384
|
+
if (opts.dryRun) {
|
|
385
|
+
if (opts.json) {
|
|
386
|
+
print(JSON.stringify({
|
|
387
|
+
dryRun: true, input: plan.label, output: plan.output, estimate,
|
|
388
|
+
files: plan.files.map((f) => ({
|
|
389
|
+
path: f.path, source: f.source, status: f.status, ...(f.note ? { note: f.note } : {}),
|
|
390
|
+
passages: (f.units || []).map((u) => u.text),
|
|
391
|
+
})),
|
|
392
|
+
}, null, 2));
|
|
393
|
+
return EXIT_OK;
|
|
394
|
+
}
|
|
395
|
+
print(`Dry run: nothing is sent and nothing is written.\n${where}\n`);
|
|
396
|
+
for (const f of plan.files) {
|
|
397
|
+
if (f.status === "copy") { print(`${f.path}: copied unchanged (${f.note})`); continue; }
|
|
398
|
+
const chars = f.units.reduce((n, u) => n + u.text.length, 0);
|
|
399
|
+
print(`${f.path}: ${plural(f.units.length, "passage", "passages")}, ${fmt(chars)} characters${f.status === "over-budget" ? " (over --max-chars, not sent)" : ""}`);
|
|
400
|
+
f.units.forEach((u, k) => print(` ${k + 1}. ${preview(u.text)}`));
|
|
401
|
+
}
|
|
402
|
+
print(`\n${estimate.join("\n")}`);
|
|
403
|
+
return EXIT_OK;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
if (!refining().length) {
|
|
407
|
+
log(`${where}\nNothing to refine: no passages in ${plural(plan.files.length, "file", "files")}.`);
|
|
408
|
+
} else {
|
|
409
|
+
log(`${where}\n${estimate.join("\n")}`);
|
|
410
|
+
if (!opts.yes) {
|
|
411
|
+
const interactive = io.interactive ?? Boolean(process.stdin.isTTY && process.stderr.isTTY);
|
|
412
|
+
if (!interactive) {
|
|
413
|
+
log("Nothing was sent. Without a terminal to ask in, pass --yes to confirm (see the estimate above, or run with --dry-run first).");
|
|
414
|
+
return EXIT_ERROR;
|
|
415
|
+
}
|
|
416
|
+
const confirm = io.confirm || (async (q) => (await import("./update.mjs")).ask(q));
|
|
417
|
+
const n = refining().reduce((k, f) => k + f.units.length, 0);
|
|
418
|
+
if (!(await confirm(`Send ${plural(n, "passage", "passages")} to Altalandi?`))) {
|
|
419
|
+
log("Nothing was sent.");
|
|
420
|
+
return EXIT_ERROR;
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// Never write inside the input, whatever the paths say.
|
|
426
|
+
const write = async (target, data) => {
|
|
427
|
+
if (plan.input && (samePath(target, plan.input) || isInside(target, plan.input))) {
|
|
428
|
+
throw new Error(`refusing to write inside the input: ${target}`);
|
|
429
|
+
}
|
|
430
|
+
if (plan.outputIsFolder ? !isInside(target, plan.output) : !samePath(target, plan.output)) {
|
|
431
|
+
throw new Error(`refusing to write outside the output: ${target}`);
|
|
432
|
+
}
|
|
433
|
+
await mkdir(path.dirname(target), { recursive: true });
|
|
434
|
+
await writeFile(target, data);
|
|
435
|
+
};
|
|
436
|
+
|
|
437
|
+
const state = { stop: null, pausedUntil: 0, errorsInARow: 0 };
|
|
438
|
+
const stop = (reason, message) => { if (!state.stop) state.stop = { reason, message }; };
|
|
439
|
+
|
|
440
|
+
for (const f of plan.files.filter((x) => x.status === "copy")) {
|
|
441
|
+
try { await write(f.target, f.bytes ?? f.text); } catch (e) { stop("error", `Could not write ${f.target}: ${e.message}`); }
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
const requestOptions = {
|
|
445
|
+
apiKey: opts.key, baseUrl, domain: opts.domain, register: opts.register,
|
|
446
|
+
// Only verified changes are written, so ask for the check explicitly.
|
|
447
|
+
preserveMeaning: true,
|
|
448
|
+
...(opts.noStore ? { contribute: false } : {}),
|
|
449
|
+
...(io.fetch ? { fetch: io.fetch } : {}),
|
|
450
|
+
};
|
|
451
|
+
|
|
452
|
+
const pause = async () => {
|
|
453
|
+
for (let left = state.pausedUntil - now(); left > 0; left = state.pausedUntil - now()) await wait(left);
|
|
454
|
+
};
|
|
455
|
+
|
|
456
|
+
const callOnce = async (call) => {
|
|
457
|
+
let rateLimited = 0;
|
|
458
|
+
let transient = 0;
|
|
459
|
+
for (;;) {
|
|
460
|
+
await pause();
|
|
461
|
+
if (state.stop) return { skipped: true };
|
|
462
|
+
try {
|
|
463
|
+
return { data: await refine(call.text, requestOptions) };
|
|
464
|
+
} catch (e) {
|
|
465
|
+
if (e.status === 429) {
|
|
466
|
+
rateLimited++;
|
|
467
|
+
const ms = Number.isFinite(e.retryAfterMs) ? e.retryAfterMs : Math.min(60_000, 2000 * 2 ** (rateLimited - 1));
|
|
468
|
+
if (ms > MAX_RETRY_WAIT_MS) {
|
|
469
|
+
stop("credit", `${e.message} Try again in about ${Math.ceil(ms / 60_000)} min.`);
|
|
470
|
+
return { skipped: true };
|
|
471
|
+
}
|
|
472
|
+
if (rateLimited > MAX_RATE_LIMIT_RETRIES) {
|
|
473
|
+
stop("error", `Still rate limited after ${MAX_RATE_LIMIT_RETRIES} retries: ${e.message}`);
|
|
474
|
+
return { skipped: true };
|
|
475
|
+
}
|
|
476
|
+
if (now() + ms > state.pausedUntil) {
|
|
477
|
+
state.pausedUntil = now() + ms;
|
|
478
|
+
log(`Rate limited (429): waiting ${Math.ceil(ms / 1000)} s, as the server asked.`);
|
|
479
|
+
}
|
|
480
|
+
continue;
|
|
481
|
+
}
|
|
482
|
+
if (e.status === 402) { stop("credit", e.message); return { skipped: true }; }
|
|
483
|
+
if (e.status === 401 || e.status === 403) { stop("error", `${e.message} → The key is invalid or revoked. Run: npx altalandi login`); return { skipped: true }; }
|
|
484
|
+
// Network errors, timeouts and server errors get two more tries.
|
|
485
|
+
if ((!e.status || e.status === 408 || e.status >= 500) && transient < MAX_TRANSIENT_RETRIES) {
|
|
486
|
+
transient++;
|
|
487
|
+
await wait(1000 * 2 ** transient);
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
return { error: e.message };
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
};
|
|
494
|
+
|
|
495
|
+
const progress = (file) => {
|
|
496
|
+
const u = file.units;
|
|
497
|
+
const changed = u.filter((x) => x.status === "changed").length;
|
|
498
|
+
const kept = u.filter((x) => x.status === "kept").length;
|
|
499
|
+
const failed = u.filter((x) => x.status === "failed").length;
|
|
500
|
+
const extra = [kept && `${kept} kept as they were (check did not pass)`, failed && `${failed} failed`].filter(Boolean).join(", ");
|
|
501
|
+
log(`${kept || failed ? "!" : "✓"} ${file.path}: ${changed} of ${plural(u.length, "passage", "passages")} changed${extra ? `; ${extra}` : ""}`);
|
|
502
|
+
};
|
|
503
|
+
|
|
504
|
+
const finish = async (file) => {
|
|
505
|
+
file.status = "writing"; // before the first await, so no other worker writes it too
|
|
506
|
+
const values = new Map(file.units.filter((u) => u.status === "changed").map((u) => [u, u.value]));
|
|
507
|
+
try {
|
|
508
|
+
await write(file.target, values.size ? assemble(file.text, file.units, values) : (file.bytes ?? file.text));
|
|
509
|
+
file.status = values.size ? "written" : "unchanged";
|
|
510
|
+
progress(file);
|
|
511
|
+
} catch (e) {
|
|
512
|
+
file.status = "failed";
|
|
513
|
+
file.note = e.message;
|
|
514
|
+
stop("error", `Could not write ${file.target}: ${e.message}`);
|
|
515
|
+
}
|
|
516
|
+
};
|
|
517
|
+
|
|
518
|
+
const done = async (call) => {
|
|
519
|
+
if (call.result.skipped) return;
|
|
520
|
+
if (call.result.error) {
|
|
521
|
+
state.errorsInARow++;
|
|
522
|
+
if (state.errorsInARow >= ERRORS_IN_A_ROW_TO_STOP) stop("error", `${state.errorsInARow} calls in a row failed; the last: ${call.result.error}`);
|
|
523
|
+
} else {
|
|
524
|
+
state.errorsInARow = 0;
|
|
525
|
+
}
|
|
526
|
+
for (const unit of call.units) {
|
|
527
|
+
if (unit.status || !unit.calls.every((c) => c.result && !c.result.skipped)) continue;
|
|
528
|
+
settle(unit);
|
|
529
|
+
const file = unit.file;
|
|
530
|
+
if (file.status === "refine" && file.units.every((u) => u.status)) await finish(file);
|
|
531
|
+
}
|
|
532
|
+
};
|
|
533
|
+
|
|
534
|
+
const hadWork = refining().length > 0;
|
|
535
|
+
const queue = [...calls];
|
|
536
|
+
const worker = async () => {
|
|
537
|
+
while (queue.length && !state.stop) {
|
|
538
|
+
const call = queue.shift();
|
|
539
|
+
call.result = await callOnce(call);
|
|
540
|
+
await done(call);
|
|
541
|
+
}
|
|
542
|
+
};
|
|
543
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(1, calls.length)) }, worker));
|
|
544
|
+
|
|
545
|
+
for (const f of plan.files) if (f.status === "refine") f.status = "unfinished";
|
|
546
|
+
|
|
547
|
+
// ── results ───────────────────────────────────────────────────────────────
|
|
548
|
+
const files = plan.files.map((f) => ({
|
|
549
|
+
path: f.path,
|
|
550
|
+
...(f.source && isUrl ? { source: f.source } : {}),
|
|
551
|
+
output: display(f.target, cwd),
|
|
552
|
+
status: f.status,
|
|
553
|
+
...(f.note ? { note: f.note } : {}),
|
|
554
|
+
passages: (f.units || []).map((u, k) => passageSummary(u, k + 1)),
|
|
555
|
+
}));
|
|
556
|
+
const passages = files.flatMap((f) => f.passages);
|
|
557
|
+
const count = (s) => passages.filter((p) => p.status === s).length;
|
|
558
|
+
const summary = {
|
|
559
|
+
input: plan.label,
|
|
560
|
+
output: display(plan.output, cwd),
|
|
561
|
+
mode: plan.mode,
|
|
562
|
+
domain: opts.domain,
|
|
563
|
+
register: opts.register,
|
|
564
|
+
finishedAt: new Date(now()).toISOString(),
|
|
565
|
+
stopped: state.stop,
|
|
566
|
+
totals: {
|
|
567
|
+
files: files.length,
|
|
568
|
+
written: files.filter((f) => f.status === "written").length,
|
|
569
|
+
unchanged: files.filter((f) => f.status === "unchanged").length,
|
|
570
|
+
copied: files.filter((f) => f.status === "copy").length,
|
|
571
|
+
unfinished: files.filter((f) => f.status === "unfinished").length,
|
|
572
|
+
overBudget: files.filter((f) => f.status === "over-budget").length,
|
|
573
|
+
passages: passages.length,
|
|
574
|
+
changed: count("changed"),
|
|
575
|
+
unchangedPassages: count("unchanged"),
|
|
576
|
+
kept: count("kept"),
|
|
577
|
+
failed: count("failed"),
|
|
578
|
+
characters: calls.filter((c) => c.result && !c.result.skipped).reduce((n, c) => n + c.text.length, 0),
|
|
579
|
+
},
|
|
580
|
+
...(plan.skipped?.length ? { skippedPages: plan.skipped } : {}),
|
|
581
|
+
files,
|
|
582
|
+
};
|
|
583
|
+
|
|
584
|
+
let exit = EXIT_OK;
|
|
585
|
+
if (summary.totals.kept) exit = EXIT_KEPT;
|
|
586
|
+
if (summary.totals.failed || summary.totals.overBudget || state.stop) exit = EXIT_ERROR;
|
|
587
|
+
if (state.stop?.reason === "credit") exit = EXIT_CREDIT;
|
|
588
|
+
summary.exitCode = exit;
|
|
589
|
+
|
|
590
|
+
if (passages.length) {
|
|
591
|
+
// Next to the output by default (./altalandi-reports for `altalandi ./docs`),
|
|
592
|
+
// and never inside the input, where the next run would pick it up.
|
|
593
|
+
const beside = path.join(path.dirname(plan.output), "altalandi-reports");
|
|
594
|
+
let reportDir = path.resolve(cwd, opts.reportDir || process.env.ALTALANDI_REPORT_DIR || beside);
|
|
595
|
+
if (plan.input && (samePath(reportDir, plan.input) || isInside(reportDir, plan.input))) reportDir = beside;
|
|
596
|
+
try {
|
|
597
|
+
summary.report = await writeBatchReport(summary, reportDir);
|
|
598
|
+
} catch (e) {
|
|
599
|
+
log(`Could not write the report: ${e.message}`);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
const t = summary.totals;
|
|
604
|
+
if (state.stop) {
|
|
605
|
+
log(`\nStopped: ${state.stop.message}`);
|
|
606
|
+
if (state.stop.reason === "credit") log("→ Add credit at https://altalandi.is/stjornbord/inneign, then run the same command for the rest.");
|
|
607
|
+
const finished = files.filter((f) => ["written", "unchanged"].includes(f.status)).map((f) => f.path);
|
|
608
|
+
const unfinished = files.filter((f) => f.status === "unfinished").map((f) => f.path);
|
|
609
|
+
log(`Finished (${finished.length}): ${finished.join(", ") || "none"}`);
|
|
610
|
+
log(`Not finished, not written (${unfinished.length}): ${unfinished.join(", ") || "none"}`);
|
|
611
|
+
}
|
|
612
|
+
log(`\n${state.stop ? "Partly done" : "Done"}: ${plural(t.written + t.unchanged + t.copied, "file", "files")} in ${display(plan.output, cwd)}`
|
|
613
|
+
+ `${t.copied ? ` (${fmt(t.copied)} copied unchanged)` : ""}.`
|
|
614
|
+
+ (hadWork ? ` Passages: ${fmt(t.changed)} changed, ${fmt(t.unchangedPassages)} needed no change, ${fmt(t.kept)} kept as they were, ${fmt(t.failed)} failed.` : ""));
|
|
615
|
+
if (t.kept) log(`${plural(t.kept, "passage keeps", "passages keep")} the original text because a check did not pass; the report lists ${t.kept === 1 ? "it" : "them"}.`);
|
|
616
|
+
if (t.overBudget) log(`${plural(t.overBudget, "file was", "files were")} left out for --max-chars.`);
|
|
617
|
+
if (summary.report) log(`Report: ${display(summary.report, cwd)}`);
|
|
618
|
+
if (opts.json) print(JSON.stringify(summary, null, 2));
|
|
619
|
+
return exit;
|
|
620
|
+
}
|