mixdog 1.3.2 → 1.3.3

package/src/runtime/attachments/legacy-office.mjs

@@ -0,0 +1,110 @@
+// Legacy Office (.doc / .xls / .ppt, OLE compound) attachments. Text
+// extraction reads OOXML packages only, so a legacy file is saved as its
+// package by LibreOffice (convertLegacyOffice, the conversion `office open`
+// uses) in a private scratch folder, never beside the user's file.
+import { mkdtemp, readFile, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { basename, extname, join } from 'node:path';
+import { redactDirectories } from '../shared/redact-directories.mjs';
+import { removeDirectoryWithRetry } from '../shared/remove-directory.mjs';
+
+// The first eight bytes of every legacy Office (OLE compound) file.
+const OLE_COMPOUND_MAGIC = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
+
+const PACKAGE_BY_LEGACY_EXTENSION = { doc: 'docx', xls: 'xlsx', ppt: 'pptx' };
+const PACKAGE_BY_LEGACY_MIME = {
+  'application/msword': 'docx',
+  'application/vnd.ms-excel': 'xlsx',
+  'application/vnd.ms-powerpoint': 'pptx',
+};
+const PACKAGE_BY_OOXML_EXTENSION = { docx: 'docx', xlsx: 'xlsx', xlsm: 'xlsx', pptx: 'pptx' };
+
+export const OOXML_MIME_BY_PACKAGE = {
+  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+};
+const PACKAGE_BY_OOXML_MIME = {
+  ...Object.fromEntries(Object.entries(OOXML_MIME_BY_PACKAGE).map(([kind, mime]) => [mime, kind])),
+  'application/vnd.ms-excel.sheet.macroenabled.12': 'xlsx',
+};
+
+export function isLegacyOfficeBuffer(buffer) {
+  return buffer.length >= 8 && buffer.subarray(0, 8).equals(OLE_COMPOUND_MAGIC);
+}
+
+/** The package extension (`docx`/`xlsx`/`pptx`) an attachment labelled legacy Office
+ *  (by MIME type or file extension) converts to, else ''. */
+export function legacyOfficeLabelPackage(mimeType, filename) {
+  const mime = String(mimeType || '')
+    .split(';', 1)[0]
+    .trim()
+    .toLowerCase();
+  const extension = extname(String(filename || ''))
+    .slice(1)
+    .toLowerCase();
+  return PACKAGE_BY_LEGACY_MIME[mime] || PACKAGE_BY_LEGACY_EXTENSION[extension] || '';
+}
+
+/** The package an OLE file labelled as an OOXML Office file (renamed, or sent
+ *  with the OOXML MIME type and no extension) converts to, else ''. */
+export function ooxmlLabelPackage(mimeType, filename) {
+  const mime = String(mimeType || '')
+    .split(';', 1)[0]
+    .trim()
+    .toLowerCase();
+  const extension = extname(String(filename || ''))
+    .slice(1)
+    .toLowerCase();
+  return PACKAGE_BY_OOXML_MIME[mime] || PACKAGE_BY_OOXML_EXTENSION[extension] || '';
+}
+
+async function defaultConvert(path, output, options) {
+  const { convertLegacyOffice } = await import('../office/portable/portable-soffice.mjs');
+  return convertLegacyOffice(path, output, options);
+}
+
+/**
+ * Saves the legacy Office `buffer` as the OOXML package `kind` and returns the
+ * package bytes and its file name. The scratch folder is always removed, once
+ * the converter has returned (it returns only after its process has closed;
+ * a removal that still meets a held file is retried, and a final failure is
+ * logged). A failure (LibreOffice missing, damaged or protected file, bytes
+ * that are not a legacy Office file) throws a message with the scratch path
+ * taken out; a cancellation rethrows the abort itself.
+ */
+export async function convertLegacyOfficeBuffer(
+  buffer,
+  filename,
+  kind,
+  { signal = null, convert = defaultConvert, remove = undefined, log = undefined, retryMs = undefined } = {}
+) {
+  signal?.throwIfAborted();
+  if (!isLegacyOfficeBuffer(buffer)) {
+    throw new Error(
+      `${filename || 'This attachment'} is labelled as a legacy Office file but its content is not one, so it cannot be converted. Open it in Microsoft Office and save it as .${kind}, then attach that copy.`
+    );
+  }
+  const stem = basename(String(filename || 'attachment'), extname(String(filename || ''))) || 'attachment';
+  const safeStem = stem.replace(/[^\w.-]+/g, '_').slice(0, 80) || 'attachment';
+  const legacyExtension = Object.keys(PACKAGE_BY_LEGACY_EXTENSION).find(
+    (ext) => PACKAGE_BY_LEGACY_EXTENSION[ext] === kind
+  );
+  const dir = await mkdtemp(join(tmpdir(), 'mixdog-attachment-'));
+  let converted;
+  try {
+    const source = join(dir, `${safeStem}.${legacyExtension}`);
+    const output = join(dir, `${safeStem}.${kind}`);
+    await writeFile(source, buffer);
+    await convert(source, output, { signal });
+    converted = { buffer: await readFile(output), filename: `${stem}.${kind}` };
+  } catch (error) {
+    signal?.throwIfAborted();
+    throw new Error(redactDirectories(error instanceof Error ? error.message : String(error), [dir]));
+  } finally {
+    await removeDirectoryWithRetry(dir, { remove, log, retryMs });
+  }
+  // A cancel that arrived while the folder was being removed still cancels.
+  signal?.throwIfAborted();
+  return converted;
+}

package/src/runtime/attachments/prompt-restore.mjs

@@ -0,0 +1,70 @@
+// Rebuilds the composer's restore payload (`pastedTexts`, `pastedImages`) from a
+// stored, materialized user prompt. The model history holds the EXPANDED prompt:
+// each `[Pasted text #n …]` / `[File #n …]` token became a `textKind` reference
+// part, in token order, and each image an image reference part. The transcript
+// row's own text still spells the tokens, so the n-th reference part belongs to
+// the n-th token; the entries carry references only and are hydrated by the
+// caller (`hydratePastedAttachments`) exactly as the abort/queue restores are.
+import { isAttachmentReference } from './store.mjs';
+
+const TEXT_TOKEN_RE = /\[(Pasted text|File) #(\d+)(?:: ([^\]\r\n]+))?[^\]\r\n]*\]/g;
+const IMAGE_TOKEN_RE = /\[Image #(\d+)[^\]\r\n]*\]/g;
+
+export function restorableAttachmentsFromPrompt(content, rowText) {
+  const parts = Array.isArray(content) ? content : [];
+  const text = String(rowText ?? '');
+  const textRefs = parts.filter(
+    (part) =>
+      part?.type === 'text' && isAttachmentReference(part) && (part.textKind === 'paste' || part.textKind === 'file')
+  );
+  const imageRefs = parts.filter((part) => part?.type === 'image' && isAttachmentReference(part));
+
+  // A `[File #n: name]` token is also spelled for binary files (PDF/Office),
+  // which never became a text reference: a token takes the next text reference
+  // only when it is the same kind (and, for files, the same name).
+  const pastedTexts = {};
+  let next = 0;
+  for (const match of text.matchAll(TEXT_TOKEN_RE)) {
+    const part = textRefs[next];
+    if (!part) break;
+    const isFile = match[1] === 'File';
+    if (isFile ? part.textKind !== 'file' || match[3] !== part.filename : part.textKind !== 'paste') continue;
+    next += 1;
+    const id = Number(match[2]);
+    if (!id || pastedTexts[id]) continue;
+    pastedTexts[id] = {
+      id,
+      source: part.textKind === 'file' ? 'file' : 'paste',
+      attachmentRef: part.attachmentRef,
+      sizeBytes: part.sizeBytes,
+      ...(part.filename ? { filename: part.filename } : {}),
+      ...(part.mimeType ? { mimeType: part.mimeType } : {}),
+    };
+  }
+
+  const imageIds = [...text.matchAll(IMAGE_TOKEN_RE)].map((match) => Number(match[1]));
+  const usedIds = new Set([...Object.keys(pastedTexts).map(Number), ...imageIds]);
+  let nextId = 1;
+  const pastedImages = {};
+  imageRefs.forEach((part, index) => {
+    let id = Number(part.pasteId) || imageIds[index] || 0;
+    if (!id || pastedImages[id]) {
+      while (usedIds.has(nextId)) nextId += 1;
+      id = nextId;
+      usedIds.add(id);
+    }
+    pastedImages[id] = {
+      id,
+      type: 'image',
+      mediaType: String(part.mimeType || part.mediaType || 'image/png'),
+      attachmentRef: part.attachmentRef,
+      sizeBytes: part.sizeBytes,
+      ...(part.filename ? { filename: part.filename } : {}),
+    };
+  });
+
+  return {
+    pastedTexts: Object.keys(pastedTexts).length ? pastedTexts : null,
+    pastedImages: Object.keys(pastedImages).length ? pastedImages : null,
+  };
+}

package/src/runtime/attachments/store.mjs

@@ -25,6 +25,13 @@ import { dirname, join } from 'node:path';
 
 import { writeFileAtomicSync, writeJsonAtomicAsync } from '../shared/atomic-file.mjs';
 import { resolvePluginData } from '../shared/plugin-paths.mjs';
+import {
+  convertLegacyOfficeBuffer,
+  isLegacyOfficeBuffer,
+  legacyOfficeLabelPackage,
+  OOXML_MIME_BY_PACKAGE,
+  ooxmlLabelPackage,
+} from './legacy-office.mjs';
 import { MAX_ATTACHMENT_BLOB_BYTES, MAX_PDF_BYTES, MAX_PDF_PAGES, isPdfBuffer } from './limits.mjs';
 import { inspectPdfBuffer, PDF_INTAKE_TIMEOUT_MS } from './pdf-extract.mjs';
 
@@ -389,7 +396,7 @@ function materializePastedTexts(pastedTexts) {
         type: 'text',
         attachmentRef: raw.attachmentRef,
         sizeBytes: attachmentBlobSize(raw, raw.sizeBytes),
-        ...(raw.source === 'file' ? { textKind: 'file' } : {}),
+        textKind: raw.source === 'file' ? 'file' : 'paste',
         ...(raw.filename ? { filename: raw.filename } : {}),
         ...(raw.mimeType ? { mimeType: raw.mimeType } : {}),
       };
@@ -630,6 +637,50 @@ export function providerTakesNativePdf(provider) {
   return provider?.nativePdf !== false;
 }
 
+/**
+ * A legacy Office attachment (.doc/.xls/.ppt: by label, or an OLE file renamed
+ * to an OOXML name) as the OOXML package LibreOffice saves it as, stored under
+ * its own reference, so text extraction and every later lowering see an
+ * ordinary package. A conversion that cannot run (LibreOffice missing, damaged
+ * file, or bytes that are not a legacy file under a legacy label) becomes a text
+ * part carrying the converter's message, so the model and the user are told
+ * instead of the file vanishing. A cancellation (`signal`) is rethrown, never
+ * turned into text. Any other part is returned as is.
+ */
+async function packageLegacyOfficePart(part, { convertLegacy, signal }) {
+  if (part?.type !== 'file' || !isAttachmentReference(part)) return part;
+  const mimeType = part.mimeType || part.mediaType;
+  const legacyKind = legacyOfficeLabelPackage(mimeType, part.filename);
+  const kind = legacyKind || ooxmlLabelPackage(mimeType, part.filename);
+  if (!kind) return part;
+  const buffer = readAttachmentBuffer(part);
+  // An OOXML-labelled part converts only when its bytes are legacy; a legacy
+  // label is always converted (or explained).
+  if (!legacyKind && !isLegacyOfficeBuffer(buffer)) return part;
+  const name = String(part.filename || '')
+    .split(/[\\/]/)
+    .pop();
+  try {
+    const converted = await convertLegacyOfficeBuffer(buffer, part.filename, kind, {
+      signal,
+      ...(convertLegacy ? { convert: convertLegacy } : {}),
+    });
+    const { mediaType: _mediaType, ...rest } = part;
+    return {
+      ...rest,
+      ...saveBuffer(converted.buffer),
+      mimeType: OOXML_MIME_BY_PACKAGE[kind],
+      filename: converted.filename,
+    };
+  } catch (error) {
+    if (signal?.aborted) throw error;
+    return {
+      type: 'text',
+      text: `--- ${name || 'attachment'} ---\n${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}
+
 /**
  * PDF intake validation. Every PDF keeps its content-addressed reference — the
  * one conversion to text happens at provider lowering — so this only checks what
@@ -638,13 +689,19 @@ export function providerTakesNativePdf(provider) {
  * PDF as text reads its first MAX_PDF_PAGES pages, so a longer PDF is accepted.
  * The page count is recorded on the part for estimates.
  */
-export async function preparePromptSubmissionForProvider(intake, provider) {
+export async function preparePromptSubmissionForProvider(
+  intake,
+  provider,
+  { convertLegacy = undefined, signal = null } = {}
+) {
   const prompt = intake?.prompt;
   if (!Array.isArray(prompt)) return intake;
   const nativePdf = providerTakesNativePdf(provider);
   let changed = false;
   const prepared = [];
-  for (const part of prompt) {
+  for (const candidate of prompt) {
+    const part = await packageLegacyOfficePart(candidate, { convertLegacy, signal });
+    if (part !== candidate) changed = true;
     const mimeType = String(part?.mimeType || part?.mediaType || '').toLowerCase();
     if (part?.type !== 'file' || mimeType !== 'application/pdf' || !isAttachmentReference(part)) {
       prepared.push(part);

package/src/runtime/effort-judge/judge-client.mjs

@@ -14,7 +14,12 @@ import { fileURLToPath } from 'node:url';
 import { resolvePluginData } from '../shared/plugin-paths.mjs';
 import { onnxRuntimeSupported } from '../shared/onnx-runtime-support.mjs';
 import { fileHasData } from '../shared/file-has-data.mjs';
-import { effortJudgeInstallCurrent, effortJudgeInstallStamped, installEffortJudgeModel } from './model-install.mjs';
+import {
+  effortJudgeInstallCurrent,
+  effortJudgeInstallStamped,
+  installEffortJudgeModel,
+  invalidateDamagedEffortJudgeInstall,
+} from './model-install.mjs';
 
 export const EFFORT_JUDGE_UNSUPPORTED = 'Auto effort is unsupported on darwin-x64 (no onnxruntime-node binding)';
 const WORKER_PATH = fileURLToPath(new URL('./judge-worker.mjs', import.meta.url));
@@ -139,6 +144,7 @@ function ensureWorker(dir) {
       lastLoadError = msg.message;
       lastLoadErrorAt = Date.now();
       notifyReady(false);
+      void repairAfterLoadError(dir);
     } else if (msg.type === 'result') {
       settle(msg.id, (entry) => entry.resolve(msg.probs));
     } else if (msg.type === 'error') {
@@ -204,6 +210,49 @@ function request(target, { request: text, prev, prevRequest, step }) {
 // One install at a time (the settings install and the boot refresh can overlap).
 let installing = null;
 let refreshChecked = false;
+let verifying = false;
+
+/** Why the judge last failed to load; '' when it loaded or never failed. */
+export function effortJudgeLastError() {
+  return lastLoadError;
+}
+
+// Runs `task` as the one install: the slot stays taken until it settles, so
+// installs, refreshes and repairs never overlap.
+function occupyInstall(task) {
+  const run = task().finally(() => {
+    if (installing === run) installing = null;
+  });
+  installing = run;
+  return run;
+}
+
+// A model that fails to load may be damaged on disk (a crash can leave a
+// stamped file with a zeroed tail). Re-verify the hashes once per load error,
+// off the turn's path: a mismatch drops the stamp and re-downloads only the bad
+// files; matching hashes keep the plain LOAD_RETRY_MS back-off. Verifying and
+// invalidating hold the install slot, after any install in flight has settled,
+// so the stamp they remove is never one an installer just wrote.
+async function repairAfterLoadError(dir) {
+  if (verifying || process.env.MIXDOG_EFFORT_JUDGE_DIR || !onnxRuntimeSupported()) return;
+  verifying = true;
+  try {
+    while (installing) await installing.catch(() => {});
+    await occupyInstall(async () => {
+      const bad = await invalidateDamagedEffortJudgeInstall(dir);
+      if (!bad.length) return;
+      process.stderr.write(`[effort-judge] reinstalling damaged model files: ${bad.join(', ')}\n`);
+      await runInstall();
+      // The files are replaced: the next turn loads them without the back-off.
+      lastLoadErrorAt = 0;
+      lastLoadError = '';
+    });
+  } catch (error) {
+    process.stderr.write(`[effort-judge] model repair failed: ${error?.message || error}\n`);
+  } finally {
+    verifying = false;
+  }
+}
 
 /**
  * Installs or updates the judge model from the release the bundled manifest
@@ -213,19 +262,17 @@ let refreshChecked = false;
  */
 export function installEffortJudge() {
   if (!onnxRuntimeSupported()) return Promise.reject(new Error(EFFORT_JUDGE_UNSUPPORTED));
-  installing ??= (async () => {
-    const dir = effortJudgeModelDir();
-    if (process.env.MIXDOG_EFFORT_JUDGE_DIR || (effortJudgeAvailable(dir) && effortJudgeInstallCurrent(dir))) {
-      if (!effortJudgeAvailable(dir))
-        throw new Error(`The Auto reasoning model is not installed (expected in ${dir}).`);
-      return;
-    }
-    await installEffortJudgeModel(dir);
-    if (worker) await shutdownEffortJudge();
-  })().finally(() => {
-    installing = null;
-  });
-  return installing;
+  return installing ?? occupyInstall(runInstall);
+}
+
+async function runInstall() {
+  const dir = effortJudgeModelDir();
+  if (process.env.MIXDOG_EFFORT_JUDGE_DIR || (effortJudgeAvailable(dir) && effortJudgeInstallCurrent(dir))) {
+    if (!effortJudgeAvailable(dir)) throw new Error(`The Auto reasoning model is not installed (expected in ${dir}).`);
+    return;
+  }
+  await installEffortJudgeModel(dir);
+  if (worker) await shutdownEffortJudge();
 }
 
 /**

package/src/runtime/effort-judge/model-install.mjs

@@ -5,9 +5,10 @@
 // install.json records the installed release; a newer bundled manifest makes
 // the install stale and the next refresh downloads only what changed.
 import { existsSync, mkdirSync, readFileSync, renameSync, rmSync } from 'node:fs';
+import { open as openAsync } from 'node:fs/promises';
 import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { parseJsonText, writeJsonAtomicSync } from '../shared/atomic-file.mjs';
+import { isFileFsyncUnsupported, parseJsonText, writeJsonAtomicSync } from '../shared/atomic-file.mjs';
 import { downloadToFileWithRetry } from '../shared/bounded-download.mjs';
 import { sha256File, validSha256 } from '../shared/native-asset.mjs';
 
@@ -52,10 +53,40 @@ export function effortJudgeInstallCurrent(dir, manifest = effortJudgeManifest())
   return stampedRelease(dir) === manifest.release;
 }
 
+/**
+ * Files in `dir` that are missing or differ from the manifest sha256. When any
+ * does, the stamp is removed so the folder reads as not installed and the next
+ * install re-downloads only those files (a crash can leave a stamped model with
+ * its tail zeroed: same size, wrong hash).
+ */
+export async function invalidateDamagedEffortJudgeInstall(dir, manifest = effortJudgeManifest()) {
+  const bad = [];
+  for (const [name, file] of Object.entries(manifest.files)) {
+    const target = join(dir, name);
+    if (!existsSync(target) || (await sha256File(target)) !== file.sha256) bad.push(name);
+  }
+  if (bad.length) rmSync(join(dir, STAMP), { force: true });
+  return bad;
+}
+
+// Flushes the downloaded file through a write handle (Windows refuses fsync on
+// a read-only one) so a crash right after the rename cannot publish a file
+// whose size is committed but whose tail never reached the disk.
+export async function fsyncFile(path, open = openAsync) {
+  const handle = await open(path, 'r+');
+  try {
+    await handle.sync();
+  } catch (err) {
+    if (!isFileFsyncUnsupported(err)) throw err;
+  } finally {
+    await handle.close();
+  }
+}
+
 /** Download what is missing or different, then stamp the release. */
 export async function installEffortJudgeModel(
   dir,
-  { fetchFn = globalThis.fetch, manifest = effortJudgeManifest() } = {}
+  { fetchFn = globalThis.fetch, manifest = effortJudgeManifest(), syncFile = fsyncFile } = {}
 ) {
   mkdirSync(dir, { recursive: true });
   let replaced = false;
@@ -73,6 +104,7 @@ export async function installEffortJudgeModel(
       const actual = await sha256File(partial);
       if (actual !== file.sha256)
         throw new Error(`effort judge ${name}: sha256 mismatch (expected ${file.sha256}, got ${actual})`);
+      await syncFile(partial);
       // From the first replaced file until the new stamp, the folder is a mix of
       // releases: without a stamp it reads as not installed, so an interrupted
       // install is never loaded.

package/src/runtime/memory/lib/memory-chunk-quality.mjs

@@ -86,11 +86,23 @@ export function parseCycle1LineFormat(raw) {
   const chunks = [];
   for (const line of lines) {
     const parts = line.split('|');
-    if (parts.length < 4 || !/^\d+(?:\s*,\s*\d+)*$/.test(parts[0].trim())) {
+    // Models often write a run as `1-8` (sessions 2026-10-22: 32 of 71 lines);
+    // an ascending range expands to its indexes, everything else stays invalid.
+    if (parts.length < 4 || !/^\d+(?:\s*-\s*\d+)?(?:\s*,\s*\d+(?:\s*-\s*\d+)?)*$/.test(parts[0].trim())) {
       chunks.push({ _idxList: [], _parseError: 'invalid_line_format' });
       continue;
     }
-    const indexes = parts[0].split(',').map((value) => Number(value.trim()));
+    const indexes = [];
+    for (const token of parts[0].split(',')) {
+      const [from, to = from] = token.split('-').map((value) => Number(value.trim()));
+      // Past the safe-integer range `n + 1 === n`, so the expansion would never end.
+      if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to) || to < from || to - from >= 1000) {
+        indexes.length = 0;
+        indexes.push(0);
+        break;
+      }
+      for (let n = from; n <= to; n += 1) indexes.push(n);
+    }
     chunks.push({
       _idxList: indexes,
       element: parts[1].trim(),