altalandi 0.6.2 → 0.6.4

package/README.md

@@ -99,6 +99,8 @@ altalandi --fast "..."                 # skip the meaning + nativeness gates
 altalandi --diff --file draft.txt      # show before, after and every change with its reason
 altalandi --review --file in.txt --out out.txt   # for agents: one block to show as it is
 altalandi --report --file draft.txt    # also write a Markdown report to ./altalandi-reports/
+altalandi --review --job j_20261013a1b2c3 --job-title "Handbók" --part "Hluti 1 af 3" --file part1.txt
+                                       # one part of a longer text: same --job on every part
 ```
 
 | flag | default | meaning |
@@ -112,6 +114,9 @@ altalandi --report --file draft.txt    # also write a Markdown report to ./altal
 | `--out <file>` | | write the refined text to `<file>` (UTF-8); with `--review` the block names it when the text is shortened |
 | `--report` | off | also write `./altalandi-reports/altalandi-<time>-<id>.md`: input, output, a table of every change (before, after, reason, layer), warnings and the verdict |
 | `--report-dir <dir>` | | write the report to `<dir>` instead (implies `--report`) |
+| `--job <id>` | | make this call a part of a job (verk): every call with the same id is one job on the site, one page for the whole text. The id is yours to choose: `j_` and 8–64 of `A-Z a-z 0-9 _ -`, e.g. `j_` + the date + a few random characters. With `--review`, the block links the job (`Verkið`) and this part (`Hlutinn`) |
+| `--job-title <text>` | | the job's name (up to 120 characters); the first call that sends one names it |
+| `--part <text>` | | this call's part, e.g. `Hluti 2 af 5` (up to 300 characters); needs `--job` |
 | `--fast` | off | `preserveMeaning:false`, `scoreNativeness:false`, `onlyImprove:false`, `polishPass:false`, `scoring:"none"` — normally one model call (structured input can still add a structure-preserving retry) |
 
 ## Folders, i18n catalogs and websites
@@ -182,11 +187,20 @@ agents) it sends nothing unless you pass `--yes`.
 | `--site` / `--max-pages <n>` | off / `20` | read a site from the URL, up to n pages |
 | `--no-store` | off | keep none of the passages (free use is otherwise kept, scrubbed of personal data) |
 | `--json` | off | a machine-readable summary of every file and passage on stdout |
+| `--job <id>` / `--job-title <text>` | a new job | add the run to an existing job, or name it |
 
 A `429` waits as long as `Retry-After` asks (up to 5 minutes) and retries. A
 `402` stops the run cleanly: finished files are written, the rest are listed and
 not written; run the same command again after topping up.
 
+**One job.** A run is one job (verk) on your account: every request sends the
+same `job` id (`j_` + the date + 10 random characters, made per run), titled by
+the folder's name or the site's host, and each file (its relative path) or page
+(its URL) is a part. A batch that holds several small files is named by its
+first file; a file that fits in one batch is not split across two. The summary
+and the report show one `Verkið: <link>` to the job's page, not one link per
+call. An engine that does not know jobs simply leaves the line out.
+
 **Results.** Only verified changes are written. A passage keeps its original
 text when its meaning check did not pass (`meaningCheck` other than `passed`, or
 `meaningPreserved` not `true`), or when the refined text lost or changed code, a

package/altalandi.mjs

@@ -108,6 +108,9 @@ for (let i = 0; i < args.length; i++) {
   else if (a === "--yes" || a === "-y") opts.yes = true;
   else if (a === "--max-chars") opts.maxChars = Number(args[++i]);
   else if (a === "--concurrency") opts.concurrency = Number(args[++i]);
+  else if (a === "--job") opts.job = args[++i] ?? "";
+  else if (a === "--job-title") opts.jobTitle = args[++i] ?? "";
+  else if (a === "--part") opts.part = args[++i] ?? "";
   else positional.push(a);
 }
 
@@ -122,6 +125,9 @@ Usage:
                                         for agents: one block to show as it is
   echo "text" | altalandi
   altalandi --domain marketing --register warm "text"
+  altalandi --review --job j_20261013a1b2c3 --job-title "Handbók" --part "Hluti 1 af 3" --file part1.txt
+                                        one part of a longer text: every part with the same
+                                        --job is one verk (job) on the site
 
 Folders and websites:
   altalandi ./docs                      every .md .txt .html .json under ./docs → ./docs-is
@@ -154,6 +160,10 @@ Options:
   --report     also write a Markdown report (input, output, changes, reasons, verdict)
                to ./altalandi-reports/  (or --report-dir <dir>; ALTALANDI_REPORT=1 turns it on)
   --file       read the text from a UTF-8 file instead of an argument
+  --job        a job id (j_ and 8-64 of A-Z a-z 0-9 _ -) you choose: calls with the same id
+               are the parts of one verk; --review then links the job and this part
+  --job-title  the job's name, kept from the first call that sends one (up to 120 characters)
+  --part       this call's part, e.g. "Hluti 2 af 5" (up to 300 characters; needs --job)
   --base-url   API base  (default $ALTALANDI_BASE_URL or https://altalandi.is/api)
 
 stdout = refined text; stderr = nativeness / meaning / warnings / diff.
@@ -169,6 +179,9 @@ Folder and website options:
   --site         with a URL: its pages on the same site, under the URL's folder
   --max-pages    pages to read with --site (default 20)
   --json         print a machine-readable summary at the end
+  --job          add the run to this job; --job-title names it (default: a new
+                 job per run, named by the folder or the site's host; each file
+                 or page is a part)
 Code, URLs, link targets, placeholders ({name}, %s, ICU), HTML tags and JSON keys
 stay byte for byte. Only changes whose meaning check passed are written; other
 passages keep the original text and are listed in the report
@@ -186,6 +199,13 @@ function readStdin() {
   });
 }
 
+// --job / --job-title / --part: checked here so a bad id never reaches the API.
+const JOB_ID = /^j_[A-Za-z0-9_-]{8,64}$/;
+if (opts.job !== undefined && !JOB_ID.test(opts.job)) {
+  console.error(`--job must be j_ followed by 8 to 64 letters, digits, _ or - (for example j_20261013a1b2c3); got "${opts.job}".`);
+  process.exit(2);
+}
+
 const stored = readCredentials();
 const baseUrl = (opts.baseUrl || process.env.ALTALANDI_BASE_URL || stored?.baseUrl || "https://altalandi.is/api").replace(/\/$/, "");
 const { key } = resolveKey({ flag: opts.key });
@@ -206,6 +226,10 @@ if (opts.site || opts.dryRun || opts.maxChars !== undefined || opts.maxPages !==
   console.error("--site, --dry-run, --max-chars, --max-pages and --concurrency need a folder or a URL:  altalandi ./docs --dry-run");
   process.exit(2);
 }
+if (opts.job === undefined && (opts.jobTitle !== undefined || opts.part !== undefined)) {
+  console.error("--job-title and --part need --job <id>: the same id on every part of the text.");
+  process.exit(2);
+}
 
 let text = positional.join(" ").trim();
 if (opts.file) {
@@ -224,6 +248,14 @@ const requestOptions = {
   register: opts.register,
   ...(opts.noStore ? { contribute: false } : {}),
 };
+if (opts.job) {
+  const cap = (s, n) => [...String(s)].slice(0, n).join("");
+  requestOptions.job = {
+    id: opts.job,
+    ...(opts.jobTitle ? { title: cap(opts.jobTitle, 120) } : {}),
+    ...(opts.part ? { part: cap(opts.part, 300) } : {}),
+  };
+}
 // --fast disables every optional quality/verification pass. The writer can
 // still retry once when it drops document structure, as documented in --help.
 // Name every switch explicitly so a future server-default change cannot make

package/batch.mjs

@@ -8,6 +8,9 @@
 //                printed before anything is sent
 //   3. consent   a y/N question in a terminal, or --yes; with neither, nothing
 //                is sent (--dry-run stops here on purpose)
+//      classify  one free POST /v1/jobs with a sample cut across every passage,
+//                so the engine knows the kind of text and its fields; a failure
+//                (or an older engine's 404) is ignored and the run goes on
 //   4. refine    short passages go together in batch requests (/v1/refine/batch),
 //                longer ones one call each; 2 requests at a time by default; a
 //                429 waits for Retry-After, a 402 stops the run cleanly
@@ -19,12 +22,13 @@
 // Every side effect is injectable (io), so the tests need no network, terminal
 // or API key.
 
+import { randomInt } from "node:crypto";
 import { existsSync, statSync } from "node:fs";
 import { mkdir, readdir, readFile, realpath, writeFile } from "node:fs/promises";
 import path from "node:path";
 
-import { refine, refineBatch } from "./client.mjs";
-import { CHARS_PER_WORD, charsEquivalent, formatChars, formatIsk, writeBatchReport } from "./report.mjs";
+import { createJob, refine, refineBatch } from "./client.mjs";
+import { CHARS_PER_WORD, charsEquivalent, formatChars, formatIsk, jobTypeText, writeBatchReport } from "./report.mjs";
 import { assemble, FORMATS, isConfigJson, repairCodeSpans, segment, segmentMarkdown, splitForApi } from "./segment.mjs";
 import { pagePath, readPage, readSite, safeSegment, userAgent } from "./web.mjs";
 
@@ -52,6 +56,16 @@ export const BATCH_MAX_CHARS = 6000;
 // nor the word units of an older engine.
 const MIN_BILLABLE_CHARS = 250;
 const BATCH_ITEM_MIN_CHARS = 130;
+// One run is one job ("verk") on the account: every request names it, and each
+// file or page is a part. The engine keeps a title of 120 and a part of 300.
+export const JOB_ID_PATTERN = /^j_[A-Za-z0-9_-]{8,64}$/;
+const JOB_TITLE_MAX = 120;
+const JOB_PART_MAX = 300;
+// The sample a run sends to POST /v1/jobs (the engine takes up to 12,000).
+export const JOB_SAMPLE_CHARS = 3000;
+const JOB_SAMPLE_MIN_PIECE = 300; // a long passage is cut to no less than this
+const JOB_SAMPLE_MAX_PIECES = 80;
+const JOB_CLASSIFY_TIMEOUT_MS = 20_000;
 const DOMAINS = ["general", "marketing", "data-analysis", "childrens-fiction"];
 const REGISTERS = ["formal", "neutral", "warm"];
 
@@ -73,6 +87,45 @@ const preview = (s, n = 72) => {
   return one.length > n ? `${one.slice(0, n - 1)}…` : one;
 };
 
+/** A new job id: `j_`, the UTC date (yyyymmdd) and 10 random [a-z0-9]. */
+export function newJobId(now = Date.now()) {
+  const abc = "abcdefghijklmnopqrstuvwxyz0123456789";
+  const date = new Date(now).toISOString().slice(0, 10).replace(/-/g, "");
+  return `j_${date}${Array.from({ length: 10 }, () => abc[randomInt(abc.length)]).join("")}`;
+}
+
+/** At most n code points, so a cut never splits a surrogate pair. */
+const cap = (s, n) => [...String(s)].slice(0, n).join("");
+
+/** At most n code points of s, cut back to a word boundary when one is near. */
+function cutAtWord(s, n) {
+  const cps = [...s];
+  if (cps.length <= n) return s;
+  const head = cps.slice(0, n).join("");
+  const space = head.search(/\s\S*$/);
+  return (space >= n / 2 ? head.slice(0, space) : head).trim();
+}
+
+/**
+ * The sample for POST /v1/jobs: passages spread evenly across the whole job
+ * (the first and the last always among them), about `budget` characters in
+ * all and never more. Each distinct passage counts once. When everything fits
+ * it is sent whole; otherwise as many evenly spaced passages as fit, each cut
+ * to an equal share (at least JOB_SAMPLE_MIN_PIECE characters, at a word).
+ */
+export function jobSample(texts, budget = JOB_SAMPLE_CHARS) {
+  const all = [...new Set(texts.map((t) => String(t).replace(/\s+/g, " ").trim()).filter(Boolean))];
+  const len = (t) => [...t].length;
+  if (all.reduce((n, t) => n + len(t), 0) <= budget) return all;
+  const spread = (m) => (m === 1 ? [0] : Array.from({ length: m }, (_, j) => Math.round((j * (all.length - 1)) / (m - 1))));
+  for (let m = Math.min(all.length, JOB_SAMPLE_MAX_PIECES); m >= 1; m--) {
+    const share = Math.max(JOB_SAMPLE_MIN_PIECE, Math.floor(budget / m));
+    const picked = [...new Set(spread(m))].map((i) => cutAtWord(all[i], Math.min(share, budget)));
+    if (picked.reduce((n, t) => n + len(t), 0) <= budget) return picked.filter(Boolean);
+  }
+  return [cutAtWord(all[0], budget)];
+}
+
 /** Billable characters: every Unicode code point of the trimmed text, spaces,
  *  punctuation and line breaks included (the server's rule). */
 export const billableChars = (text) => [...String(text).trim()].length;
@@ -159,9 +212,10 @@ async function planFolder(opts, io) {
       }
     }
     file.status = file.note ? "copy" : "refine";
+    file.part = rel;
     files.push(file);
   }
-  return { mode: "folder", label: opts.target, input, output, outputIsFolder: true, files };
+  return { mode: "folder", label: opts.target, title: path.basename(given), input, output, outputIsFolder: true, files };
 }
 
 const withSource = (page) => `<!-- Source: ${page.url} -->\n\n${page.markdown}\n`;
@@ -170,7 +224,7 @@ function pageFile(page, rel, target) {
   const text = withSource(page);
   const { units } = segmentMarkdown(text);
   return {
-    path: rel, source: page.url, target, format: "md", text, units,
+    path: rel, source: page.url, part: page.url, target, format: "md", text, units,
     status: units.length ? "refine" : "copy", note: units.length ? undefined : "nothing to refine",
   };
 }
@@ -184,7 +238,7 @@ async function planWeb(opts, io, log) {
     const name = `${last === "index" ? safeSegment(new URL(page.url).host) : last}-is.md`;
     let target = path.resolve(cwd, opts.out || name);
     if (existsSync(target) && statSync(target).isDirectory()) target = path.join(target, name);
-    return { mode: "page", label: opts.target, output: target, outputIsFolder: false, files: [pageFile(page, path.basename(target), target)] };
+    return { mode: "page", label: opts.target, title: new URL(page.url).host, output: target, outputIsFolder: false, files: [pageFile(page, path.basename(target), target)] };
   }
   log(`Reading ${opts.target} (up to ${plural(opts.maxPages, "page", "pages")}) …`);
   const { pages, skipped, source } = await readSite(opts.target, {
@@ -199,7 +253,7 @@ async function planWeb(opts, io, log) {
     taken.add(rel.toLowerCase());
     return pageFile(page, rel, path.join(output, ...rel.split("/")));
   });
-  return { mode: "site", label: opts.target, output, outputIsFolder: true, files, skipped, discovery: source };
+  return { mode: "site", label: opts.target, title: new URL(pages[0].url).host, output, outputIsFolder: true, files, skipped, discovery: source };
 }
 
 // ── calls ───────────────────────────────────────────────────────────────────
@@ -225,7 +279,8 @@ function buildCalls(files, maxChars) {
       continue;
     }
     budgetUsed += added;
-    for (const [text, call] of fresh) calls.set(text, call);
+    // A call belongs to the first file that sends it: that file is its part.
+    for (const [text, call] of fresh) calls.set(text, Object.assign(call, { part: file.part }));
     for (const unit of file.units) {
       unit.file = file;
       for (const call of new Set(unit.calls)) call.units.push(unit);
@@ -235,18 +290,36 @@ function buildCalls(files, maxChars) {
 }
 
 /** The requests of a run: short calls together in batches of at most
- *  BATCH_MAX_ITEMS items and BATCH_MAX_CHARS characters, longer ones alone. */
+ *  BATCH_MAX_ITEMS items and BATCH_MAX_CHARS characters, longer ones alone.
+ *  A batch takes whole files: when a file's short calls fit in a batch of
+ *  their own but not in the room left in the open one, that batch is closed
+ *  and the file starts the next, so a file is split only when it alone is
+ *  bigger than a batch. Small files still share a batch (named by the first
+ *  file's part). The limits are unchanged, and closing a batch early costs
+ *  nothing: two or more items are already past the call minimum. */
 export function groupCalls(calls) {
-  const jobs = [];
+  const requests = [];
   let open = null;
-  for (const call of calls) {
-    if (call.text.length > BATCH_PASSAGE_MAX_CHARS) { jobs.push({ batch: false, calls: [call] }); continue; }
-    if (open && (open.calls.length >= BATCH_MAX_ITEMS || open.chars + call.text.length > BATCH_MAX_CHARS)) open = null;
-    if (!open) { open = { batch: true, calls: [], chars: 0 }; jobs.push(open); }
-    open.calls.push(call);
-    open.chars += call.text.length;
+  const isShort = (c) => c.text.length <= BATCH_PASSAGE_MAX_CHARS;
+  for (let i = 0; i < calls.length;) {
+    let end = i;
+    while (end < calls.length && calls[end].part === calls[i].part) end++;
+    const file = calls.slice(i, end);
+    i = end;
+    const short = file.filter(isShort);
+    const chars = short.reduce((n, c) => n + c.text.length, 0);
+    const fitsAlone = short.length <= BATCH_MAX_ITEMS && chars <= BATCH_MAX_CHARS;
+    if (open && short.length && fitsAlone
+      && (open.calls.length + short.length > BATCH_MAX_ITEMS || open.chars + chars > BATCH_MAX_CHARS)) open = null;
+    for (const call of file) {
+      if (!isShort(call)) { requests.push({ batch: false, calls: [call] }); continue; }
+      if (open && (open.calls.length >= BATCH_MAX_ITEMS || open.chars + call.text.length > BATCH_MAX_CHARS)) open = null;
+      if (!open) { open = { batch: true, calls: [], chars: 0 }; requests.push(open); }
+      open.calls.push(call);
+      open.chars += call.text.length;
+    }
   }
-  return jobs;
+  return requests;
 }
 
 /** Characters a request is billed: the call minimum for one text; for a batch,
@@ -413,6 +486,11 @@ export async function runBatch(opts, io = {}) {
   const maxPages = opts.maxPages ?? 20;
   if (!Number.isInteger(maxPages) || maxPages < 1) { log("--max-pages must be a positive whole number."); return EXIT_ERROR; }
   if (opts.site && !isUrl) { log("--site needs a URL, such as https://example.com"); return EXIT_ERROR; }
+  if (opts.part !== undefined) { log("--part is for one text. A folder or site run names each part by its file or page."); return EXIT_ERROR; }
+  if (opts.job !== undefined && !JOB_ID_PATTERN.test(String(opts.job))) {
+    log("--job must be j_ followed by 8 to 64 letters, digits, _ or - (for example j_20261013a1b2c3).");
+    return EXIT_ERROR;
+  }
   // Checked here, not by the API: a bad value would otherwise fail every call.
   if (!DOMAINS.includes(opts.domain ?? "general")) { log(`--domain must be one of: ${DOMAINS.join(", ")}`); return EXIT_ERROR; }
   if (!REGISTERS.includes(opts.register ?? "neutral")) { log(`--register must be one of: ${REGISTERS.join(", ")}`); return EXIT_ERROR; }
@@ -519,6 +597,22 @@ export async function runBatch(opts, io = {}) {
     ...(opts.noStore ? { contribute: false } : {}),
     ...(io.fetch ? { fetch: io.fetch } : {}),
   };
+  // One job for the whole run (--job joins an existing one); each request names
+  // its part: the file's relative path, or the page URL.
+  const runJob = {
+    id: opts.job || newJobId(now()),
+    title: cap(opts.jobTitle || plan.title || plan.label || "", JOB_TITLE_MAX),
+  };
+  const withPart = (part) => ({
+    ...requestOptions,
+    job: { id: runJob.id, ...(runJob.title ? { title: runJob.title } : {}), ...(part ? { part: cap(part, JOB_PART_MAX) } : {}) },
+  });
+  // The job's page, from the first response that names it (an older engine never does).
+  const noteJob = (data) => {
+    if (state.jobUrl) return;
+    const url = [data?.jobUrl, data?.items?.[0]?.jobUrl].find((u) => typeof u === "string" && u);
+    if (url) Object.assign(state, { jobUrl: url, jobId: typeof data.jobId === "string" && data.jobId ? data.jobId : runJob.id });
+  };
 
   const pause = async () => {
     for (let left = state.pausedUntil - now(); left > 0; left = state.pausedUntil - now()) await wait(left);
@@ -615,18 +709,21 @@ export async function runBatch(opts, io = {}) {
   };
 
   const single = async (call) => {
-    const r = await attempt(() => refine(call.text, requestOptions));
-    if (r.data) bill(r.data);
+    const r = await attempt(() => refine(call.text, withPart(call.part)));
+    if (r.data) { bill(r.data); noteJob(r.data); }
     return r;
   };
 
   /** One batch request; every item is mapped back to its call by id. */
   const sendBatch = async (job) => {
-    const items = job.calls.map((c, k) => ({ id: String(k), text: c.text }));
-    const r = await attempt(() => refineBatch(items, requestOptions), { batch: true });
+    // Each item names its own file (or page): a batch that holds several small
+    // files reports each one on the job page. The request is named by its first.
+    const items = job.calls.map((c, k) => ({ id: String(k), text: c.text, ...(c.part ? { part: cap(c.part, JOB_PART_MAX) } : {}) }));
+    const r = await attempt(() => refineBatch(items, withPart(job.calls[0].part)), { batch: true });
     if (!r.data) return r;
     const data = r.data;
     bill(data);
+    noteJob(data);
     const byId = new Map(data.items.map((it) => [String(it.id), it]));
     const artifacts = data.quality?.encodingArtifacts || [];
     job.calls.forEach((call, k) => {
@@ -642,6 +739,28 @@ export async function runBatch(opts, io = {}) {
     return { done: true };
   };
 
+  // Before the first refine call: one free POST /v1/jobs with a sample of the
+  // whole run, so the engine knows the kind of text and its fields for every
+  // call. Never stops the run: an older engine answers 404, and any error is
+  // ignored (the engine then classifies from the first call's own text).
+  if (calls.length && !state.stop) {
+    // Every passage of every file that is sent (not the ones left out by --max-chars).
+    const texts = refining().flatMap((f) => f.units.map((u) => u.text));
+    try {
+      const data = await createJob({
+        id: runJob.id, ...(runJob.title ? { title: runJob.title } : {}), sample: jobSample(texts),
+      }, { apiKey: opts.key, baseUrl, timeoutMs: JOB_CLASSIFY_TIMEOUT_MS, ...(io.fetch ? { fetch: io.fetch } : {}) });
+      const c = data.classification;
+      if (c && typeof c === "object") {
+        state.classification = c;
+        log(`Tegund: ${jobTypeText(c) || "óviss"}`);
+      }
+      if (typeof data.jobUrl === "string" && data.jobUrl) Object.assign(state, { jobUrl: data.jobUrl, jobId: data.id || runJob.id });
+    } catch {
+      // 404 (no jobs on this engine), a network error or a timeout: carry on.
+    }
+  }
+
   const alone = (job) => job.calls.map((call) => ({ batch: false, calls: [call] }));
   const hadWork = refining().length > 0;
   const queue = [...jobs];
@@ -724,6 +843,8 @@ export async function runBatch(opts, io = {}) {
       units: state.billed,
     },
     ...(plan.skipped?.length ? { skippedPages: plan.skipped } : {}),
+    ...(state.jobUrl ? { jobId: state.jobId, jobUrl: state.jobUrl } : {}),
+    ...(state.classification ? { classification: state.classification } : {}),
     files,
   };
 
@@ -758,6 +879,8 @@ export async function runBatch(opts, io = {}) {
   log(`\n${state.stop ? "Partly done" : "Done"}: ${plural(t.written + t.unchanged + t.copied, "file", "files")} in ${display(plan.output, cwd)}`
     + `${t.copied ? ` (${fmt(t.copied)} copied unchanged)` : ""}.`
     + (hadWork ? ` Passages: ${fmt(t.changed)} changed, ${fmt(t.unchangedPassages)} needed no change, ${fmt(t.kept)} kept as they were, ${fmt(t.failed)} failed.` : ""));
+  // One link for the whole run, never one per call.
+  if (summary.jobUrl) log(`Verkið: ${summary.jobUrl}`);
   if (t.billedCharacters != null) {
     log(`Rukkað: ${t.billedCharactersApprox ? "um " : ""}${formatChars(t.billedCharacters)} (${charsEquivalent(t.billedCharacters)}).`);
   }