artifacty 0.10.8 → 0.11.0

package/src/lib/backup.js

@@ -1,11 +1,64 @@
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
 import path from "node:path";
-import { createStore, loadIndex, writeIndex } from "./storage.js";
+import {
+  countUsers,
+  createStore,
+  exportFullStoreTables,
+  importFullStoreTables,
+  loadIndex,
+  openDatabase,
+  STORE_VERSION,
+  writeIndex
+} from "./storage.js";
 
-export async function exportStore(store = createStore(), outputPath) {
+export const MAX_BACKUP_BYTES = 128 * 1024 * 1024;
+const BUNDLE_VERSION = 2;
+
+export async function exportStore(store = createStore(), outputPath, { scope = "artifacts" } = {}) {
   if (!outputPath) {
     throw new Error("export requires --file <path>");
   }
+  const normalizedScope = scope === "full" ? "full" : "artifacts";
+  const bundle = await buildStoreBackup(store, { scope: normalizedScope });
+  const resolvedOutputPath = path.resolve(outputPath);
+  await mkdir(path.dirname(resolvedOutputPath), { recursive: true });
+  const content = `${JSON.stringify(bundle, null, 2)}\n`;
+  if (normalizedScope === "full") {
+    // The full scope carries password hashes and API token hashes. Write to
+    // a private temp file (mode 0o600 from creation, "wx" so it fails
+    // instead of following a symlink or clobbering something already
+    // there) and rename it into place, so the destination is never briefly
+    // readable with the process's default (typically 0o644) mode the way a
+    // plain writeFile-then-chmod leaves it for the duration of the write.
+    const tempPath = `${resolvedOutputPath}.${randomUUID()}.tmp`;
+    try {
+      await writeFile(tempPath, content, { encoding: "utf8", mode: 0o600, flag: "wx" });
+      await rename(tempPath, resolvedOutputPath);
+    } catch (error) {
+      // Don't leave a stray 0o600 temp file containing password/API-token
+      // hashes behind if the rename (or the write itself) fails.
+      await rm(tempPath, { force: true });
+      throw error;
+    }
+  } else {
+    await writeFile(resolvedOutputPath, content, "utf8");
+  }
+  return {
+    path: resolvedOutputPath,
+    artifactCount: bundle.artifacts.length,
+    scope: normalizedScope,
+    exportedAt: bundle.exportedAt
+  };
+}
+
+export async function exportStoreToString(store = createStore(), { scope = "artifacts" } = {}) {
+  const bundle = await buildStoreBackup(store, { scope: scope === "full" ? "full" : "artifacts" });
+  return `${JSON.stringify(bundle, null, 2)}\n`;
+}
+
+export async function buildStoreBackup(store = createStore(), { scope = "artifacts" } = {}) {
+  const normalizedScope = scope === "full" ? "full" : "artifacts";
   const index = await loadIndex(store);
   const artifacts = [];
   for (const artifact of index.artifacts) {
@@ -13,6 +66,7 @@ export async function exportStore(store = createStore(), outputPath) {
     for (const version of artifact.versions) {
       versions.push({
         ...version,
+        path: normalizeBackupRelativePath(version.path),
         content: await readFile(path.join(store.home, version.path), "utf8")
       });
     }
@@ -20,26 +74,121 @@ export async function exportStore(store = createStore(), outputPath) {
   }
 
   const bundle = {
+    bundleVersion: BUNDLE_VERSION,
+    scope: normalizedScope,
+    storeVersion: STORE_VERSION,
     schemaVersion: 1,
     exportedAt: new Date().toISOString(),
     artifacts
   };
-  await mkdir(path.dirname(path.resolve(outputPath)), { recursive: true });
-  await writeFile(outputPath, `${JSON.stringify(bundle, null, 2)}\n`, "utf8");
-  return {
-    path: path.resolve(outputPath),
-    artifactCount: artifacts.length,
-    exportedAt: bundle.exportedAt
-  };
+
+  if (normalizedScope === "full") {
+    bundle.full = await exportFullStoreTables(store);
+  }
+
+  return bundle;
 }
 
-export async function importStore(store = createStore(), inputPath) {
+export async function importStore(store = createStore(), inputPath, options = {}) {
   if (!inputPath) {
     throw new Error("import-store requires --file <path>");
   }
   const bundle = JSON.parse(await readFile(inputPath, "utf8"));
+  const result = await importStoreBundle(store, bundle, options);
+  return {
+    path: path.resolve(inputPath),
+    ...result
+  };
+}
+
+export async function importStoreFromString(store = createStore(), content, options = {}) {
+  if (!String(content || "").trim()) {
+    throw new Error("Artifacty backup JSON is required");
+  }
+  let bundle;
+  try {
+    bundle = JSON.parse(content);
+  } catch (error) {
+    throw Object.assign(new Error(`Invalid Artifacty backup JSON: ${error.message}`), {
+      statusCode: 400,
+      code: "INVALID_BACKUP_JSON"
+    });
+  }
+  return importStoreBundle(store, bundle, options);
+}
+
+export async function importStoreBundle(store = createStore(), bundle, { confirm, forceUsers = false } = {}) {
+  if (!bundle || typeof bundle !== "object") {
+    throw Object.assign(new Error("Invalid Artifacty backup: JSON object expected"), {
+      statusCode: 400,
+      code: "INVALID_BACKUP"
+    });
+  }
   if (!Array.isArray(bundle.artifacts)) {
-    throw new Error("Invalid Artifacty backup: artifacts array missing");
+    throw Object.assign(new Error("Invalid Artifacty backup: artifacts array missing"), {
+      statusCode: 400,
+      code: "INVALID_BACKUP"
+    });
+  }
+
+  // Bundles written before bundleVersion 2 (roadmap section 13) have no
+  // `scope` field; they were always artifacts-only.
+  const scope = bundle.scope === "full" ? "full" : "artifacts";
+  const warnings = [];
+
+  if (scope === "full") {
+    if (confirm !== "replace-all") {
+      throw Object.assign(new Error('Full-scope restore requires confirm: "replace-all"'), {
+        statusCode: 400,
+        code: "confirm_required"
+      });
+    }
+    if (!forceUsers) {
+      const existingUsers = await countUsers(store);
+      if (existingUsers > 0) {
+        throw Object.assign(new Error("Target store already has users; pass forceUsers to overwrite them"), {
+          statusCode: 409,
+          code: "users_exist"
+        });
+      }
+    }
+    // Validate bundle.full's shape *before* writeIndex below replaces the
+    // artifacts table and deletes version files: importFullStoreTables runs
+    // after that point, and a malformed bundle.full throwing partway
+    // through it would otherwise leave the store with the backup's
+    // artifacts but the *target's* original users/tokens/comments — an
+    // unrecoverable half-restore. This check doesn't guarantee
+    // importFullStoreTables can't still fail for other reasons (disk
+    // errors, etc.), but it catches the common case of a malformed or
+    // hand-edited bundle up front.
+    assertValidFullBackupTables(bundle.full);
+    if (!Array.isArray(bundle.full?.embeddings)) {
+      warnings.push(
+        "Backup predates embeddings in full-scope backups; run `artifacty index rebuild --embeddings` after this restore to rebuild semantic search."
+      );
+    }
+  } else {
+    // writeIndex's DELETE FROM artifacts cascades to every existing
+    // artifact's comments/relations/embeddings (foreign keys, ON DELETE
+    // CASCADE) — but an artifacts-scope bundle never carries or restores
+    // those tables. Silently wiping a target store's comment/relation
+    // history because someone ran an artifacts-only restore would be a
+    // surprising, unrecoverable data loss, so refuse unless the caller
+    // explicitly opts in the same way a full-scope restore does.
+    const dependents = countArtifactDependents(store);
+    if (dependents.comments > 0 || dependents.relations > 0) {
+      if (confirm !== "replace-all") {
+        throw Object.assign(
+          new Error(
+            'Target store has comments and/or relations that this artifacts-scope restore would delete without restoring equivalents. Pass confirm: "replace-all" to proceed anyway, or use a full-scope backup.'
+          ),
+          { statusCode: 409, code: "dependents_exist" }
+        );
+      }
+      warnings.push(
+        `Target store's ${dependents.comments} comment(s) and ${dependents.relations} relation(s) were deleted by this artifacts-scope restore and are not restored (only a full-scope backup carries them).`
+      );
+    }
   }
 
   const index = {
@@ -52,8 +201,9 @@ export async function importStore(store = createStore(), inputPath) {
     for (const version of artifact.versions || []) {
       const cleanVersion = { ...version };
       delete cleanVersion.content;
+      cleanVersion.path = normalizeBackupRelativePath(cleanVersion.path);
       const content = version.content || "";
-      const absolutePath = path.join(store.home, cleanVersion.path);
+      const absolutePath = backupVersionPath(store, cleanVersion.path);
       await mkdir(path.dirname(absolutePath), { recursive: true });
       await writeFile(absolutePath, content, "utf8");
       versions.push(cleanVersion);
@@ -62,14 +212,175 @@ export async function importStore(store = createStore(), inputPath) {
   }
 
   await writeIndex(store, index);
-  return {
-    path: path.resolve(inputPath),
+  await pruneUnreferencedArtifactFiles(store, index);
+
+  const result = {
     artifactCount: index.artifacts.length,
+    scope,
     importedAt: new Date().toISOString()
   };
+  if (warnings.length > 0) {
+    result.warnings = warnings;
+  }
+
+  if (scope === "full") {
+    result.tableCounts = await importFullStoreTables(store, bundle.full || {}, { forceUsers });
+  }
+
+  return result;
+}
+
+// Counts the rows an artifacts-scope restore would silently delete (via
+// writeIndex's artifacts-table replace cascading through foreign keys) but
+// never restores, so importStoreBundle can decide whether to warn/refuse.
+function countArtifactDependents(store) {
+  const db = openDatabase(store);
+  try {
+    return {
+      comments: db.prepare("SELECT COUNT(*) AS count FROM artifact_comments").get().count,
+      relations: db.prepare("SELECT COUNT(*) AS count FROM artifact_relations").get().count
+    };
+  } finally {
+    db.close();
+  }
+}
+
+const FULL_BACKUP_TABLE_ARRAY_KEYS = [
+  "users",
+  "apiTokens",
+  "auditLog",
+  "relations",
+  "webhooks",
+  "comments",
+  "savedViews",
+  "embeddings",
+  "meta"
+];
+
+// Partial, fail-fast validation of a full-scope backup's `full` object,
+// aimed at catching the common failure mode (a malformed or hand-edited
+// bundle) before writeIndex below has made any changes. Not exhaustive:
+// importFullStoreTables (storage.js) remains the source of truth for what
+// is actually importable, and can still throw for reasons this doesn't
+// check for.
+function assertValidFullBackupTables(full) {
+  if (full === undefined || full === null) {
+    return;
+  }
+  if (typeof full !== "object" || Array.isArray(full)) {
+    throw Object.assign(new Error("Invalid Artifacty backup: `full` must be an object"), {
+      statusCode: 400,
+      code: "INVALID_BACKUP"
+    });
+  }
+  for (const key of FULL_BACKUP_TABLE_ARRAY_KEYS) {
+    if (full[key] !== undefined && !Array.isArray(full[key])) {
+      throw Object.assign(new Error(`Invalid Artifacty backup: \`full.${key}\` must be an array`), {
+        statusCode: 400,
+        code: "INVALID_BACKUP"
+      });
+    }
+  }
+  for (const row of full.users || []) {
+    if (!row || typeof row !== "object" || !String(row.id || "").trim() || !String(row.email || "").trim()) {
+      throw Object.assign(new Error("Invalid Artifacty backup: each `full.users` entry needs an id and email"), {
+        statusCode: 400,
+        code: "INVALID_BACKUP"
+      });
+    }
+  }
+  for (const row of full.apiTokens || []) {
+    if (!row || typeof row !== "object" || !String(row.id || "").trim() || !String(row.userId || "").trim()) {
+      throw Object.assign(new Error("Invalid Artifacty backup: each `full.apiTokens` entry needs an id and userId"), {
+        statusCode: 400,
+        code: "INVALID_BACKUP"
+      });
+    }
+  }
 }
 
 export function defaultBackupPath(store = createStore()) {
   const stamp = new Date().toISOString().replace(/[:.]/g, "-");
   return path.join(store.home, "backups", `artifacty-${stamp}.json`);
 }
+
+function backupVersionPath(store, relativePath) {
+  const normalized = normalizeBackupRelativePath(relativePath);
+  const root = path.resolve(store.home);
+  const absolute = path.resolve(root, ...normalized.split("/"));
+  if (absolute !== root && !absolute.startsWith(`${root}${path.sep}`)) {
+    throw Object.assign(new Error(`Invalid Artifacty backup version path: ${relativePath}`), {
+      statusCode: 400,
+      code: "INVALID_BACKUP_PATH"
+    });
+  }
+  return absolute;
+}
+
+function normalizeBackupRelativePath(relativePath) {
+  const value = String(relativePath || "");
+  if (!value || path.isAbsolute(value)) {
+    throw Object.assign(new Error(`Invalid Artifacty backup version path: ${value}`), {
+      statusCode: 400,
+      code: "INVALID_BACKUP_PATH"
+    });
+  }
+
+  const portable = value.replaceAll("\\", "/");
+  if (/^[A-Za-z]:\//.test(portable) || portable.startsWith("//")) {
+    throw Object.assign(new Error(`Invalid Artifacty backup version path: ${value}`), {
+      statusCode: 400,
+      code: "INVALID_BACKUP_PATH"
+    });
+  }
+
+  const parts = portable
+    .split("/")
+    .filter((part) => part && part !== ".");
+  if (parts.length === 0 || parts.some((part) => part === "..")) {
+    throw Object.assign(new Error(`Invalid Artifacty backup version path: ${value}`), {
+      statusCode: 400,
+      code: "INVALID_BACKUP_PATH"
+    });
+  }
+  return parts.join("/");
+}
+
+async function pruneUnreferencedArtifactFiles(store, index) {
+  const referenced = new Set();
+  for (const artifact of index.artifacts) {
+    for (const version of artifact.versions || []) {
+      referenced.add(backupVersionPath(store, version.path));
+    }
+  }
+
+  const files = await listFiles(store.artifactsDir);
+  for (const filePath of files) {
+    if (!referenced.has(filePath)) {
+      await rm(filePath, { force: true });
+    }
+  }
+}
+
+async function listFiles(root) {
+  let entries;
+  try {
+    entries = await readdir(root, { withFileTypes: true });
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return [];
+    }
+    throw error;
+  }
+
+  const files = [];
+  for (const entry of entries) {
+    const fullPath = path.join(root, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await listFiles(fullPath));
+    } else if (entry.isFile()) {
+      files.push(path.resolve(fullPath));
+    }
+  }
+  return files;
+}

package/src/lib/converters.js

@@ -1,7 +1,7 @@
 import path from "node:path";
 import { createHash } from "node:crypto";
 import { normalizeSourceAgent } from "./agents.js";
-import { ARTIFACT_TYPES, contentTypeForFormat, normalizeFormat } from "./storage.js";
+import { ARTIFACT_TYPES, BUNDLE_BINARY_CONTENT_TYPES, contentTypeForFormat, normalizeFormat } from "./storage.js";
 
 const CONTINUATION_AGENTS = new Set(["codex", "copilot", "cursor"]);
 
@@ -119,6 +119,9 @@ export function detectFormat({ content = "", contentType = "", fileName = "" } =
   if (type.includes("vnd.ant.react") || type.includes("jsx")) {
     return "react";
   }
+  if (type.includes("ipynb") || type.includes("notebook")) {
+    return "notebook";
+  }
   if (type.includes("html")) {
     return "html";
   }
@@ -126,9 +129,12 @@ export function detectFormat({ content = "", contentType = "", fileName = "" } =
     return "markdown";
   }
   if (type.includes("json")) {
-    return "json";
+    return looksLikeNotebook(content) ? "notebook" : "json";
   }
   const lowerName = optionalString(fileName).toLowerCase();
+  if (lowerName.endsWith(".ipynb")) {
+    return "notebook";
+  }
   if (lowerName.endsWith(".sarif") || lowerName.endsWith(".sarif.json")) {
     return "sarif";
   }
@@ -161,6 +167,9 @@ export function detectFormat({ content = "", contentType = "", fileName = "" } =
   if (extension === ".jsx" || extension === ".tsx") {
     return "react";
   }
+  if (extension === ".ipynb") {
+    return "notebook";
+  }
   if (isCodeExtension(extension)) {
     return "code";
   }
@@ -190,6 +199,9 @@ export function detectFormat({ content = "", contentType = "", fileName = "" } =
   if (looksLikeSarif(trimmed)) {
     return "sarif";
   }
+  if (looksLikeNotebook(trimmed)) {
+    return "notebook";
+  }
   if (looksLikeJson(trimmed)) {
     return "json";
   }
@@ -248,6 +260,21 @@ function decodeObjectPayload(value, agent) {
     };
   }
 
+  if (isNotebookObject(artifactObject)) {
+    return {
+      content: JSON.stringify(artifactObject, null, 2),
+      format: "notebook",
+      contentType: "application/x-ipynb+json; charset=utf-8",
+      title: artifactObject.title || artifactObject.name || "Notebook",
+      sourceAgent: artifactObject.sourceAgent || artifactObject.source_agent || artifactObject.agent || (agent === "auto" ? undefined : agent),
+      artifactType: artifactObject.artifactType || artifactObject.artifact_type || "analysis-report",
+      tags: artifactObject.tags,
+      metadata: {
+        originalPayloadShape: "notebook"
+      }
+    };
+  }
+
   if (typeof artifactObject.content === "string" && !Array.isArray(artifactObject.content)) {
     const sourceContentType = artifactObject.contentType ||
       artifactObject.content_type ||
@@ -440,13 +467,30 @@ function decodeBundlePayload(object, agent) {
     ? collectContinuationMetadata(continuationAgent, { agent: sourceAgent }, bundle, object)
     : {};
   const normalizedFiles = files.map((file, index) => {
+    const filePath = optionalString(file.path || file.name || `file-${index + 1}`);
+    const declaredContentType = optionalString(file.contentType || file.mimeType).toLowerCase().split(";")[0].trim();
+    const isBinary = file.encoding === "base64" || BUNDLE_BINARY_CONTENT_TYPES.has(declaredContentType);
+
+    if (isBinary) {
+      const base64 = normalizeBase64(typeof file.content === "string" ? file.content : "");
+      const decoded = base64 ? Buffer.from(base64, "base64") : Buffer.alloc(0);
+      return {
+        path: filePath,
+        content: base64,
+        encoding: "base64",
+        contentType: declaredContentType || "application/octet-stream",
+        sizeBytes: decoded.byteLength,
+        sha256: createHash("sha256").update(decoded).digest("hex")
+      };
+    }
+
     const content = typeof file.content === "string" ? file.content : "";
     return {
-      path: optionalString(file.path || file.name || `file-${index + 1}`),
+      path: filePath,
       content,
       contentType: optionalString(file.contentType || file.mimeType) || contentTypeForFormat(detectFormat({
         content,
-        fileName: file.path || file.name
+        fileName: filePath
       })),
       sizeBytes: Buffer.byteLength(content, "utf8"),
       sha256: createHash("sha256").update(content).digest("hex")
@@ -1098,7 +1142,7 @@ function inferArtifactType({ format, content, fileName, metadata }) {
   if (format === "code") {
     return "snippet";
   }
-  if (format === "sarif") {
+  if (format === "sarif" || format === "notebook") {
     return "analysis-report";
   }
   if (format === "csv") {
@@ -1190,6 +1234,27 @@ function isSarifObject(value) {
     (typeof value.version === "string" || optionalString(value.$schema).toLowerCase().includes("sarif"));
 }
 
+function looksLikeNotebook(value) {
+  if (!looksLikeJson(value)) {
+    return false;
+  }
+  try {
+    return isNotebookObject(JSON.parse(optionalString(value)));
+  } catch {
+    return false;
+  }
+}
+
+export function isNotebookObject(value) {
+  return Boolean(
+    value &&
+      typeof value === "object" &&
+      !Array.isArray(value) &&
+      value.nbformat !== undefined &&
+      Array.isArray(value.cells)
+  );
+}
+
 function looksLikeHtml(value) {
   const trimmed = optionalString(value);
   if (/^<!doctype html/i.test(trimmed) || /^<html[\s>]/i.test(trimmed)) {
@@ -1420,3 +1485,32 @@ function optionalString(value) {
   }
   return String(value).trim();
 }
+
+// Resolves the best available image/video content type for a stored
+// artifact: an explicit media MIME type in version metadata, then the
+// version's own contentType if it is itself image/video, then a
+// format-based fallback. Used when serving raw media bytes (e.g. GET
+// /artifacts/:id/raw) so a stale or missing contentType still renders.
+export function mediaContentType(artifact) {
+  const metadataType = String(artifact.version.metadata?.mimeType || "").toLowerCase();
+  if (metadataType.startsWith("image/") || metadataType.startsWith("video/")) {
+    return metadataType;
+  }
+  const versionType = String(artifact.version.contentType || "").toLowerCase().split(";")[0];
+  if (versionType.startsWith("image/") || versionType.startsWith("video/")) {
+    return versionType;
+  }
+  return artifact.version.format === "video" ? "video/mp4" : "image/png";
+}
+
+// Decodes an image/video artifact's stored content (a data: URL or bare
+// base64 string) into raw bytes, or returns null if it isn't valid base64.
+export function decodeMediaContent(content) {
+  const value = String(content || "").trim();
+  const dataUrl = /^data:[^;,]+;base64,([A-Za-z0-9+/=_-\s]+)$/i.exec(value);
+  const base64 = (dataUrl ? dataUrl[1] : value).replace(/\s+/g, "").replaceAll("-", "+").replaceAll("_", "/");
+  if (!base64 || !/^[A-Za-z0-9+/]+={0,2}$/.test(base64)) {
+    return null;
+  }
+  return Buffer.from(base64, "base64");
+}

package/src/lib/csv.js

@@ -0,0 +1,51 @@
+// Single shared CSV parser used by the CSV renderer (render.js), the CSV
+// diff (diff.js), and the SARIF/CSV export route (sarif-csv-export.js). All
+// three previously carried byte-identical copies of this state machine that
+// had already diverged in behavior on malformed input (an unterminated
+// quoted field): the render.js copy failed closed while the diff.js copy
+// silently emitted a bogus single-cell row for the same input, so the same
+// artifact rendered as an error but diffed as data. This module is the one
+// source of truth: it always fails closed on an unterminated quoted field.
+export function parseCsv(content) {
+  const text = String(content || "");
+  const rows = [];
+  let row = [];
+  let cell = "";
+  let inQuotes = false;
+
+  for (let index = 0; index < text.length; index += 1) {
+    const char = text[index];
+    if (char === "\"") {
+      if (inQuotes && text[index + 1] === "\"") {
+        cell += "\"";
+        index += 1;
+      } else {
+        inQuotes = !inQuotes;
+      }
+    } else if (char === "," && !inQuotes) {
+      row.push(cell);
+      cell = "";
+    } else if ((char === "\n" || char === "\r") && !inQuotes) {
+      if (char === "\r" && text[index + 1] === "\n") {
+        index += 1;
+      }
+      row.push(cell);
+      if (row.length > 1 || row[0] !== "") {
+        rows.push(row);
+      }
+      row = [];
+      cell = "";
+    } else {
+      cell += char;
+    }
+  }
+
+  if (inQuotes) {
+    return { ok: false, error: "unterminated quoted field", rows: [] };
+  }
+  row.push(cell);
+  if (row.length > 1 || row[0] !== "") {
+    rows.push(row);
+  }
+  return { ok: true, rows };
+}