@cassiomc1/forgeloop 1.14.0 → 2.0.0-rc.4

package/src/repository-index/binary-manager.js

@@ -181,11 +181,18 @@ export function validateArchiveEntry(entry) {
   return normalized;
 }
 
+export function resolveArchiveTool(archive, { platform = process.platform, env = process.env } = {}) {
+  // Windows ships a ZIP-capable bsdtar. Git's GNU tar can shadow it on PATH.
+  if (platform === "win32") return path.win32.join(env.SystemRoot || "C:\\Windows", "System32", "tar.exe");
+  return archive === "zip" ? "unzip" : "tar";
+}
+
 async function listArchiveEntries(archivePath, { archive = "tar.gz", execFileImpl = nodeExecFile } = {}) {
-  const listCommand = archive === "zip" && process.platform !== "win32" ? "unzip" : "tar";
-  const listArgs = listCommand === "unzip" ? ["-Z1", archivePath] : ["-tf", archivePath];
+  const listCommand = resolveArchiveTool(archive);
+  const archiveName = path.basename(archivePath);
+  const listArgs = listCommand === "unzip" ? ["-Z1", archiveName] : ["-tf", archiveName];
   try {
-    const listed = await execFileAsync(execFileImpl, listCommand, listArgs, { maxBuffer: 2 * 1024 * 1024 });
+    const listed = await execFileAsync(execFileImpl, listCommand, listArgs, { cwd: path.dirname(archivePath), maxBuffer: 2 * 1024 * 1024 });
     return listed.stdout.split(/\r?\n/).filter(Boolean).map(validateArchiveEntry);
   } catch (cause) {
     throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXTRACTION_FAILED, `Unable to inspect tgrep archive: ${cause.message}`, { cause });
@@ -194,12 +201,13 @@ async function listArchiveEntries(archivePath, { archive = "tar.gz", execFileImp
 
 async function extractArchive(archivePath, extractionDirectory, { archive = "tar.gz", execFileImpl = nodeExecFile } = {}) {
   await listArchiveEntries(archivePath, { archive, execFileImpl });
-  const extractCommand = archive === "zip" && process.platform !== "win32" ? "unzip" : "tar";
+  const extractCommand = resolveArchiveTool(archive);
+  const destination = path.relative(path.dirname(archivePath), extractionDirectory).replaceAll("\\", "/") || ".";
   const extractArgs = extractCommand === "unzip"
-    ? ["-q", archivePath, "-d", extractionDirectory]
-    : ["-xf", archivePath, "-C", extractionDirectory];
+    ? ["-q", path.basename(archivePath), "-d", destination]
+    : ["-xf", path.basename(archivePath), "-C", destination];
   try {
-    await execFileAsync(execFileImpl, extractCommand, extractArgs, { maxBuffer: 2 * 1024 * 1024 });
+    await execFileAsync(execFileImpl, extractCommand, extractArgs, { cwd: path.dirname(archivePath), maxBuffer: 2 * 1024 * 1024 });
   } catch (cause) {
     throw repositoryIndexError(REPOSITORY_INDEX_ERROR_CODES.ENGINE_EXTRACTION_FAILED, `Unable to extract tgrep archive: ${cause.message}`, { cause });
   }

package/src/storage/artifact-bytes.js

@@ -0,0 +1,22 @@
+import { createHash } from "node:crypto";
+import { canonicalFingerprint } from "../core/artifacts.js";
+
+/** Validate byte evidence separately from the domain payload schema. */
+export function artifactByteDigest(record) {
+  if (typeof record?.sourceText !== "string" || typeof record?.byteDigest !== "string") {
+    const error = new Error("Original artifact byte evidence is unavailable");
+    error.code = "E_STORAGE_ARTIFACT_BYTES_UNAVAILABLE";
+    throw error;
+  }
+  try {
+    const digest = createHash("sha256").update(record.sourceText).digest("hex");
+    if (digest !== record.byteDigest || canonicalFingerprint(JSON.parse(record.sourceText)) !== canonicalFingerprint(record.payload)) {
+      throw new Error("Artifact source bytes, digest and canonical payload disagree");
+    }
+    return digest;
+  } catch (cause) {
+    const error = new Error("Stored artifact byte evidence is invalid", { cause });
+    error.code = "E_STORAGE_ARTIFACT_INVALID";
+    throw error;
+  }
+}

package/src/storage/attachment-binding.js

@@ -0,0 +1,11 @@
+/** Pure canonical binding validation, independent of files and SQLite. */
+export function validateAttachmentReference(reference) {
+  if (!reference || typeof reference.taskId !== "string" || !reference.taskId
+    || typeof reference.referenceId !== "string" || !reference.referenceId || reference.referenceId.length > 256
+    || !Number.isSafeInteger(reference.size) || reference.size < 0
+    || !/^[a-f0-9]{64}$/.test(reference.sha256 ?? "")
+    || reference.path !== `.forgeloop/attachments/objects/${reference.sha256}`) {
+    throw Object.assign(new Error("Invalid canonical attachment reference"), { code: "E_STORAGE_ATTACHMENT_INVALID" });
+  }
+  return reference;
+}

package/src/storage/attachment-files.js

@@ -0,0 +1,82 @@
+import { createHash } from "node:crypto";
+import { constants, createWriteStream } from "node:fs";
+import { link, lstat, mkdir, mkdtemp, open, rm } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { digestFile, syncDirectory } from "./file-durability.js";
+import { assertSafePath } from "../core/filesystem.js";
+import { assertWritableContext } from "./transaction.js";
+
+function invalid(message) { return Object.assign(new Error(message), { code: "E_STORAGE_ATTACHMENT_INVALID" }); }
+
+
+/** Verify referenced immutable bytes without loading the attachment into memory. */
+async function attachmentFilename(target, reference) {
+  if (!reference || !Number.isSafeInteger(reference.size) || reference.size < 0
+    || !/^[a-f0-9]{64}$/.test(reference.sha256 ?? "")
+    || reference.path !== `.forgeloop/attachments/objects/${reference.sha256}`) throw invalid("Invalid attachment reference");
+  const filename = await assertSafePath(target, reference.path);
+  if (!(await lstat(filename)).isFile()) throw invalid("Attachment reference must name a regular file");
+  return filename;
+}
+
+export async function verifyAttachmentFile(target, reference) {
+  const filename = await attachmentFilename(target, reference);
+  const actual = await digestFile(filename, { includeSize: true });
+  if (actual.size !== reference.size || actual.sha256 !== reference.sha256) throw invalid("Attachment bytes disagree with their reference");
+  return reference;
+}
+
+/** Give consumers private verified bytes, closing the source check/use window. */
+export async function withVerifiedAttachmentFile(target, reference, callback) {
+  const source = await attachmentFilename(target, reference);
+  const directory = await mkdtemp(path.join(os.tmpdir(), "forgeloop-attachment-read-"));
+  try {
+    const filename = path.join(directory, "bytes");
+    const handle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+      if (!(await handle.stat()).isFile()) throw invalid("Attachment must remain a regular file");
+      const hash = createHash("sha256");
+      let size = 0;
+      const check = new Transform({ transform(bytes, _encoding, done) {
+        size += bytes.length;
+        if (size > reference.size) { done(invalid("Attachment exceeds its reference size")); return; }
+        hash.update(bytes);
+        done(null, bytes);
+      } });
+      await pipeline(handle.createReadStream({ autoClose: false }), check, createWriteStream(filename, { flags: "wx", mode: 0o600 }));
+      if (size !== reference.size || hash.digest("hex") !== reference.sha256) throw invalid("Attachment bytes disagree with their reference");
+    } finally { await handle.close(); }
+    return await callback(filename);
+  } finally { await rm(directory, { recursive: true, force: true }); }
+}
+
+/** Publish immutable bytes before a caller commits their database reference. */
+export async function publishAttachmentFile(target, readable, { db, temporaryRoot } = {}) {
+  if (db?.isTransaction) throw invalid("Attachment publication must precede the database write transaction");
+  if (db) assertWritableContext(db);
+  const root = await assertSafePath(target, ".forgeloop/attachments/objects");
+  await mkdir(root, { recursive: true });
+  await assertSafePath(target, ".forgeloop/attachments/objects");
+  const temporaryParent = temporaryRoot ? await assertSafePath(temporaryRoot, ".") : root;
+  const temporary = await mkdtemp(path.join(temporaryParent, ".publishing-"));
+  const filename = path.join(temporary, "bytes");
+  try {
+    await pipeline(readable, createWriteStream(filename, { flags: "wx", mode: 0o600 }));
+    const binding = await digestFile(filename, { includeSize: true });
+    const file = await open(filename, "r+");
+    try { await file.sync(); } finally { await file.close(); }
+    const reference = { path: `.forgeloop/attachments/objects/${binding.sha256}`, ...binding };
+    const destination = await assertSafePath(target, reference.path);
+    try { await link(filename, destination); }
+    catch (error) { if (error.code !== "EEXIST") throw error; }
+    await verifyAttachmentFile(target, reference);
+    await syncDirectory(root);
+    await syncDirectory(path.dirname(root));
+    await syncDirectory(path.dirname(path.dirname(root)));
+    await syncDirectory(target);
+    return reference;
+  } finally { await rm(temporary, { recursive: true, force: true }); }
+}

package/src/storage/attachment-references.js

@@ -0,0 +1,36 @@
+import { publishAttachmentFile, verifyAttachmentFile } from "./attachment-files.js";
+import { assertWritableContext, runInTransaction } from "./transaction.js";
+import { validateAttachmentReference } from "./attachment-binding.js";
+export { validateAttachmentReference } from "./attachment-binding.js";
+
+function invalid(message) { return Object.assign(new Error(message), { code: "E_STORAGE_ATTACHMENT_INVALID" }); }
+
+/** Synchronous insertion is used only after the referenced bytes were verified. */
+export function insertVerifiedAttachmentReference(db, reference) {
+  assertWritableContext(db);
+  validateAttachmentReference(reference);
+  const prior = db.prepare("SELECT path, size, sha256 FROM attachment_references WHERE task_id = ? AND reference_id = ?").get(reference.taskId, reference.referenceId);
+  if (prior) {
+    if (prior.path !== reference.path || prior.size !== reference.size || prior.sha256 !== reference.sha256) throw invalid("An immutable attachment identity already names different bytes");
+    return reference;
+  }
+  db.prepare("INSERT INTO attachment_references(task_id, reference_id, path, size, sha256) VALUES (?, ?, ?, ?, ?)")
+    .run(reference.taskId, reference.referenceId, reference.path, reference.size, reference.sha256);
+  return reference;
+}
+
+/** Publish first, then commit the immutable binding in a short writer transaction. */
+export async function registerAttachment(db, target, { taskId, referenceId, readable }) {
+  if (db.isTransaction) throw invalid("Attachment registration cannot span an active write transaction");
+  assertWritableContext(db);
+  const binding = await publishAttachmentFile(target, readable, { db });
+  const reference = validateAttachmentReference({ taskId, referenceId, ...binding });
+  await verifyAttachmentFile(target, reference);
+  return runInTransaction(db, () => insertVerifiedAttachmentReference(db, reference));
+}
+
+export function* iterateAttachmentReferences(db) {
+  for (const row of db.prepare("SELECT * FROM attachment_references ORDER BY task_id, reference_id").iterate()) {
+    yield validateAttachmentReference({ taskId: row.task_id, referenceId: row.reference_id, path: row.path, size: row.size, sha256: row.sha256 });
+  }
+}

package/src/storage/backup.js

@@ -0,0 +1,53 @@
+import { loadStorageDriver } from "./runtime.js";
+import { mkdtemp, rm, link, open } from "node:fs/promises";
+import path from "node:path";
+import { assertSafePath } from "../core/filesystem.js";
+import { checkStorageIntegrity, openStorageDatabase } from "./connection.js";
+
+/** Database-only backup; attachment restore requires its own verified inventory. */
+export async function backupStorageDatabase(db, destination) {
+  const { backup, DatabaseSync } = loadStorageDriver();
+  if (db.isTransaction) throw Object.assign(new Error("Backup requires committed storage state"), { code: "E_STORAGE_SNAPSHOT_TRANSACTION" });
+  const absolute = path.resolve(destination);
+  const parent = path.dirname(absolute);
+  await assertSafePath(parent, path.basename(absolute));
+  const temporary = await mkdtemp(path.join(parent, ".forgeloop-backup-"));
+  const filename = path.join(temporary, "state.sqlite");
+  try {
+    await backup(db, filename);
+    const snapshot = new DatabaseSync(filename, { readOnly: true, enableForeignKeyConstraints: true });
+    try {
+      const integrity = checkStorageIntegrity(snapshot);
+      if (!integrity.ok) throw Object.assign(new Error("Backup snapshot failed storage integrity validation"), { code: "E_STORAGE_BACKUP_INVALID", integrity });
+    } finally { snapshot.close(); }
+    // FlushFileBuffers on Windows requires write access to this owned snapshot.
+    const file = await open(filename, "r+");
+    try { await file.sync(); } finally { await file.close(); }
+    // Same-directory hard-link publication is atomic and cannot overwrite a
+    // retained backup. The private temporary name is removed afterward.
+    await link(filename, absolute);
+    try {
+      const directory = await open(parent, "r");
+      try { await directory.sync(); } finally { await directory.close(); }
+    } catch (error) {
+      // Match the existing portable atomic-file boundary on platforms that
+      // cannot open/sync directories. Other I/O failures remain visible.
+      if (!["EINVAL", "EPERM", "EISDIR", "ENOTSUP", "UNKNOWN"].includes(error.code)) throw error;
+    }
+    return { path: absolute, kind: "DATABASE_ONLY", attachmentsIncluded: false };
+  } finally { await rm(temporary, { recursive: true, force: true }); }
+}
+
+/** Restore into a fresh database pathname; never replace an active project. */
+export async function restoreStorageDatabase(source, destination) {
+  const absolute = path.resolve(source);
+  const parent = path.dirname(absolute);
+  for (const suffix of ["", "-wal", "-shm"]) await assertSafePath(parent, `${path.basename(absolute)}${suffix}`);
+  // Read-only open rejects unsupported schema/format metadata rather than
+  // silently upgrading a retained backup during a restore drill.
+  const db = openStorageDatabase(absolute, { readOnly: true });
+  try {
+    const restored = await backupStorageDatabase(db, destination);
+    return { ...restored, source: absolute, restored: true };
+  } finally { db.close(); }
+}

package/src/storage/capabilities.js

@@ -0,0 +1,31 @@
+import { STORAGE_FORMAT, STORAGE_FORMAT_VERSION, STORAGE_RELATIVE_PATH, STORAGE_SCHEMA_VERSION } from "./schema.js";
+import { STORAGE_MINIMUM_NODE } from "./runtime.js";
+
+/** Package capabilities only; no project selection, database open or runtime probe. */
+export function getStorageCapabilities() {
+  return {
+    version: 1,
+    defaultBackend: "sqlite",
+    soleSQLiteWriter: false,
+    sqlite: {
+      format: STORAGE_FORMAT,
+      formatVersion: STORAGE_FORMAT_VERSION,
+      schemaVersion: STORAGE_SCHEMA_VERSION,
+      path: STORAGE_RELATIVE_PATH,
+      driver: "node:sqlite",
+      minimumNode: STORAGE_MINIMUM_NODE,
+      selection: "FRESH_WRITABLE_BOOTSTRAP_OR_EXISTING_DATABASE",
+    },
+    portableFormats: ["JSON", "NDJSON"],
+    migration: {
+      explicit: true,
+      requiresWritersQuiesced: true,
+      resumeRequiresRecordedInventory: true,
+      automaticDowngrade: false,
+      commands: ["storage-migrate", "storage-migration-resume", "storage-migration-status", "storage-rollback", "storage-rollback-resume"],
+      rollbackRequiresNativeWritesExcluded: true,
+      rollbackRequiresPinnedLegacyTarget: true,
+    },
+    backup: { database: true, referencedAttachments: true, command: "storage-backup" },
+  };
+}

package/src/storage/connection-owner.js

@@ -0,0 +1,119 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { lstat, realpath } from "node:fs/promises";
+import path from "node:path";
+
+const activeOwner = new AsyncLocalStorage();
+function closed(message) { return Object.assign(new Error(message), { code: "E_STORAGE_RUNTIME_CLOSED" }); }
+function storageChanged() { return Object.assign(new Error("Project database changed while other connection leases are active"), { code: "E_STORAGE_MIGRATION_REQUIRED" }); }
+
+async function openOwnedConnection(entry, filename, options) {
+  if (entry.db?.isTransaction && entry.active.size > 1) throw Object.assign(new Error("Another lease left an active native transaction; overlapping access is forbidden"), { code: "E_STORAGE_TRANSACTION_LEAKED" });
+  const identity = await lstat(filename, { bigint: true });
+  const sameFile = entry.identity?.dev === identity.dev && entry.identity?.ino === identity.ino;
+  if (entry.db && (!sameFile || entry.readOnly !== options.readOnly)) {
+    if (entry.active.size > 1) throw storageChanged();
+    entry.db.close(); entry.db = null;
+  }
+  if (!entry.db) {
+    const { openStorageDatabase } = await import("./connection.js");
+    const db = openStorageDatabase(filename, options);
+    const after = await lstat(filename, { bigint: true });
+    if (after.dev !== identity.dev || after.ino !== identity.ino) { db.close(); throw closed("Project database changed while opening its connection"); }
+    entry.db = db; entry.identity = after; entry.readOnly = options.readOnly;
+  }
+  const { assertStorageMetadataCurrent, assertProjectDatabaseAdmission } = await import("./connection.js");
+  assertProjectDatabaseAdmission(entry.db, options.projectAdmission);
+  assertStorageMetadataCurrent(entry.db);
+  return entry.db;
+}
+
+function connectionLease(entry, scope) {
+  return {
+    assertUnselected() {
+      if (entry.db) throw Object.assign(new Error("Previously selected project storage has lost its database; filesystem fallback is forbidden"), { code: "E_STORAGE_MIGRATION_REQUIRED" });
+    },
+    open(filename, options) {
+      // Opening/replacing a handle includes filesystem awaits. Serialize only
+      // this short admission work, never the external command callback.
+      const opening = entry.openTail.then(() => openOwnedConnection(entry, filename, options));
+      entry.openTail = opening.then(() => undefined, () => undefined);
+      return opening.then(db => { scope.opened = true; return db; });
+    },
+  };
+}
+
+async function admitLease(entry, readOnly) {
+  let release;
+  const lifetime = new Promise(resolve => { release = resolve; });
+  const admission = entry.tail.then(async () => {
+    if (entry.mode !== readOnly) await Promise.all([...entry.active]);
+    entry.mode = readOnly;
+    entry.active.add(lifetime);
+  });
+  entry.tail = admission.then(() => undefined, () => undefined);
+  await admission;
+  return () => { entry.active.delete(lifetime); release(); };
+}
+
+async function executeLease(entry, owner, callback, readOnly) {
+  const release = await admitLease(entry, readOnly);
+  const scope = { owner, active: true, opened: false };
+  try {
+    return await activeOwner.run(scope, () => callback(connectionLease(entry, scope)));
+  } finally {
+    scope.active = false;
+    try {
+      if (scope.opened && entry.db?.isTransaction) {
+        entry.db.close(); entry.db = null;
+        throw Object.assign(new Error("Command left an active native storage transaction; connection was closed"), { code: "E_STORAGE_TRANSACTION_LEAKED" });
+      }
+    } finally { release(); }
+  }
+}
+
+/** One connection per project; same-mode leases share it without external serialization. */
+export function createStorageConnectionOwner() {
+  const entries = new Map();
+  const pending = new Set();
+  let closing = false;
+  let closePromise;
+  const owner = {
+    isActive() { return activeOwner.getStore()?.owner === owner && activeOwner.getStore()?.active === true; },
+    run(target, callback, { readOnly = false } = {}) {
+      if (closing) return Promise.reject(closed("Persistent storage runtime is closing or closed"));
+      if (owner.isActive()) return Promise.reject(closed("Nested connection ownership must reuse the active operational scope"));
+      const request = (async () => {
+        let key;
+        try { key = await realpath(path.resolve(target)); }
+        catch (error) { if (error.code !== "ENOENT") throw error; key = path.resolve(target); }
+        let entry = entries.get(key);
+        if (!entry) {
+          entry = { tail: Promise.resolve(), openTail: Promise.resolve(), active: new Set(), mode: null, db: null, identity: null };
+          entries.set(key, entry);
+        }
+        return executeLease(entry, owner, callback, readOnly);
+      })();
+      pending.add(request);
+      const finished = () => { pending.delete(request); };
+      request.then(finished, finished);
+      return request;
+    },
+    close() {
+      if (owner.isActive()) return Promise.reject(closed("Close the runtime outside an active command"));
+      if (closePromise) return closePromise;
+      closing = true;
+      closePromise = (async () => {
+        await Promise.all([...pending].map(request => request.catch(() => undefined)));
+        let failure;
+        for (const entry of entries.values()) {
+          try { entry.db?.close(); } catch (error) { failure ??= error; }
+          entry.db = null;
+        }
+        entries.clear();
+        if (failure) throw failure;
+      })();
+      return closePromise;
+    },
+  };
+  return owner;
+}