@ai-ecoverse/slicc-agent 6.3.1 → 6.4.0

package/dist/freezer/v6.js

@@ -1,9 +1,28 @@
+import { replicatedState } from "@earendil-works/chord";
+import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
+import { configure } from "@earendil-works/pi-durable";
+import { freeFolder, scoopId } from "../agents.js";
+import { homeOf } from "../kernel/accounts.js";
+import { quote } from "../kernel/env.js";
+import { opfsFile } from "../opfs-pool.js";
+import { TITLE_TURNS } from "../titles/extension.js";
+import { place } from "./v6-place.js";
 import {
   checkLicks,
   checkManifest,
   checkTranscript
 } from "./v6-schema.js";
-import { ImportError, isLink, readEntry, zipEntries } from "./zip.js";
+import { Writer } from "./v6-write.js";
+import {
+  blobSource,
+  bytesSource,
+  ImportError,
+  isLink,
+  readEntry,
+  zipEntries
+} from "./zip.js";
+const V6_KIND = "slicc.v6";
+const IMPORTS_DIR = "/var/lib/slicc/agent/imports";
 const MiB = 1024 ** 2;
 const LIMITS = {
   archive: 2048 * MiB,
@@ -78,15 +97,15 @@ function scan(list, archive) {
     throw new ImportError("the zip expands over 200:1, like a zip bomb");
   return entries;
 }
-async function openProfile(bytes) {
-  if (bytes.length > LIMITS.archive) throw new ImportError("the zip is over 2 GiB");
-  const list = zipEntries(bytes);
+async function openProfile(source) {
+  if (source.size > LIMITS.archive) throw new ImportError("the zip is over 2 GiB");
+  const list = await zipEntries(source);
   if (list.length > LIMITS.entries)
     throw new ImportError(`the zip has over ${LIMITS.entries} entries`);
   if (list[0]?.name !== "manifest.json")
     throw new ImportError("manifest.json must be the first entry");
-  const entries = scan(list, bytes.length);
-  const manifest = checkManifest(json(await readEntry(bytes, list[0]), "manifest.json"));
+  const entries = scan(list, source.size);
+  const manifest = checkManifest(json(await readEntry(source, list[0]), "manifest.json"));
   const listed = /* @__PURE__ */ new Set();
   for (const file of manifest.files) {
     if (entries.get(file.path)?.size !== file.size || file.path === "manifest.json")
@@ -101,7 +120,7 @@ async function openProfile(bytes) {
     whereOf(manifest, name);
   }
   const { exportId, part } = manifest;
-  return { manifest, entries, bytes, key: part ? `${exportId}#${part.index}` : exportId };
+  return { manifest, entries, source, key: part ? `${exportId}#${part.index}` : exportId };
 }
 const noLicks = { crontasks: [], tasks: [], webhooks: [] };
 async function licksIn(opened, data) {
@@ -109,14 +128,14 @@ async function licksIn(opened, data) {
   if (!name) return noLicks;
   const entry = opened.entries.get(name);
   if (!entry) throw new ImportError(`${name} is named in the manifest but not in the zip`);
-  return checkLicks(json(data?.get(name) ?? await readEntry(opened.bytes, entry), name));
+  return checkLicks(json(data?.get(name) ?? await readEntry(opened.source, entry), name));
 }
 const UNITS = [
   [1e9, "GB"],
   [1e6, "MB"],
   [1e3, "KB"]
 ];
-async function previewOf(opened) {
+async function previewOf(opened, state) {
   const { manifest } = opened;
   const licks = await licksIn(opened);
   const bytes = manifest.files.filter((file) => file.path.startsWith("files/")).reduce((sum, file) => sum + file.size, 0);
@@ -150,6 +169,7 @@ async function previewOf(opened) {
     flagged: manifest.flagged.length,
     providers: manifest.providers,
     mounts,
+    already: state.imports?.[opened.key]?.at ?? null,
     text: `SLICC 6 profile from ${host}, exported ${day(manifest.createdAt)}: ${parts.join(", ")}. Re-enter in Settings: ${manifest.providers.join(", ") || "nothing"}. Not imported: ${plural(mounts.length, "mount")}.`
   };
 }
@@ -157,13 +177,18 @@ async function sha256(bytes) {
   const digest = await crypto.subtle.digest("SHA-256", bytes);
   return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
 }
-async function verify(opened) {
+async function verify(opened, writer, stage, show) {
   const data = /* @__PURE__ */ new Map();
-  for (const file of opened.manifest.files) {
-    const bytes = await readEntry(opened.bytes, opened.entries.get(file.path));
+  const { files } = opened.manifest;
+  for (const [at, file] of files.entries()) {
+    writer.check();
+    const bytes = await readEntry(opened.source, opened.entries.get(file.path));
     if (await sha256(bytes) !== file.sha256.toLowerCase())
       throw new ImportError(`${file.path} doesn't match its sha256`);
-    if (whereOf(opened.manifest, file.path).kind === "data") data.set(file.path, bytes);
+    const where = whereOf(opened.manifest, file.path);
+    if (where.kind === "data") data.set(file.path, bytes);
+    else if (where.kind !== "skip" && stage) await writer.put(`${stage}/${file.path}`, bytes, true);
+    show("checking", at + 1, files.length);
   }
   return data;
 }
@@ -178,6 +203,71 @@ function transcriptIn(data, name) {
     throw new ImportError(`${name}: ${error.message}`);
   }
 }
+function draft(fields, when, created) {
+  const last = fields.main?.messages.at(-1)?.timestamp;
+  const frozenAt = [...when, last, created].map((value) => Date.parse(value ?? "")).find((at) => !Number.isNaN(at));
+  const tail = (fields.doc?.conversations ?? []).map((item) => item.messages.at(-1)?.id ?? "").join(",");
+  return {
+    ...fields,
+    frozenAt,
+    tail,
+    cone: "",
+    fresh: true,
+    record: true,
+    suffix: "",
+    home: "",
+    roots: /* @__PURE__ */ new Set(),
+    files: /* @__PURE__ */ new Map()
+  };
+}
+function unitsOf(manifest, data) {
+  const cones = manifest.cones.map((cone) => {
+    const doc = transcriptIn(data, cone.transcript);
+    const all = doc?.conversations ?? [];
+    const main = all.find((item) => item.id === cone.conversation) ?? all.find((item) => item.kind === "cone");
+    const scoops = cone.scoops.map((scoop) => ({
+      name: scoop.name,
+      six: scoop.folder,
+      folder: "",
+      conversation: all.find((item) => item.id === scoop.conversation) ?? all.find((item) => item.kind === "scoop" && item.folder === scoop.folder)
+    }));
+    const title = doc?.session.title || cone.name;
+    return draft(
+      {
+        key: cone.key,
+        name: cone.name,
+        title,
+        six: cone.folder,
+        memory: cone.memory,
+        doc,
+        main,
+        scoops
+      },
+      [doc?.session.updatedAt],
+      manifest.createdAt
+    );
+  });
+  const frozen = manifest.frozen.map((item) => {
+    const doc = transcriptIn(data, item.transcript);
+    const [main, ...rest] = [...doc.conversations].sort(
+      (a, b) => Number(b.kind === "cone") - Number(a.kind === "cone")
+    );
+    const title = item.title || doc.session.title;
+    const scoops = rest.map((conversation) => ({
+      name: conversation.name,
+      six: null,
+      folder: "",
+      conversation
+    }));
+    const when = [item.frozenAt, doc.session.frozenAt, doc.session.updatedAt];
+    return draft(
+      { key: `frozen:${item.id}`, name: title, title, six: null, memory: null, doc, main, scoops },
+      when,
+      manifest.createdAt
+    );
+  });
+  return [...cones, ...frozen];
+}
 function present(data, name) {
   if (name && !data.has(name))
     throw new ImportError(`${name} is named in the manifest but not in the zip`);
@@ -216,34 +306,325 @@ function related(manifest, data) {
   }
   for (const name of transcripts) attached(manifest, name, transcriptIn(data, name));
 }
-async function checked(opened, data) {
+async function parse(opened, data) {
   for (const [name, bytes] of data)
     if (name.startsWith("memory/") || name === opened.manifest.gitconfig) textOf(bytes, name);
   related(opened.manifest, data);
-  await licksIn(opened, data);
+  return { data, licks: await licksIn(opened, data), units: unitsOf(opened.manifest, data) };
 }
-async function bytesOf(env, source, context) {
-  if ("bytes" in source) return source.bytes;
-  const info = await env.fileInfo(source.path, context);
+function prior(state, origin, key) {
+  const known = /* @__PURE__ */ new Set([
+    ...Object.keys(state.cones),
+    ...Object.values(state.frozen ?? {}).map((item) => item.cone)
+  ]);
+  return Object.values(state.imports ?? {}).filter((item) => item.origin === origin && known.has(item.cones[key])).sort((a, b) => b.at - a.at).map((item) => ({ cone: item.cones[key], tail: item.tails[key] }))[0];
+}
+async function assign(deps, opened, units, again, date, context) {
+  const { agents } = deps;
+  const origin = opened.manifest.producer.origin;
+  let fresh = 0;
+  for (const unit of units) {
+    const before = again ? void 0 : prior(agents.state(), origin, unit.key);
+    if (!before) fresh += 1;
+    if (!before) continue;
+    unit.cone = before.cone;
+    unit.fresh = false;
+    unit.record = before.tail !== unit.tail;
+    if (unit.record) unit.title = `${unit.title} (updated)`;
+    if (agents.state().cones[before.cone]) unit.suffix = `.v6-${date}`;
+  }
+  const first = fresh ? await agents.update((_tx, doc) => {
+    doc.next += fresh;
+    return doc.next - fresh;
+  }, context) : 0;
+  const taken = { ...agents.state().scoops };
+  let next = first;
+  for (const unit of units) {
+    if (unit.fresh) unit.cone = `cone-${next++}`;
+    unit.home = homeOf(unit.cone);
+    for (const scoop of unit.scoops) {
+      scoop.folder = freeFolder({ scoops: taken }, scoop.name);
+      taken[scoopId(scoop.folder)] = {};
+    }
+  }
+}
+const ZERO = {
+  input: 0,
+  output: 0,
+  cacheRead: 0,
+  cacheWrite: 0,
+  totalTokens: 0,
+  cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }
+};
+function piMessage(message, names, files) {
+  const timestamp = Date.parse(message.timestamp);
+  const texts = message.content.flatMap((part) => {
+    if (part.type === "text") return [{ type: "text", text: part.text }];
+    if (part.type === "tool-call") return [];
+    const file = files.get(part.attachmentId);
+    return [
+      {
+        type: "text",
+        text: file ? `[Attached file: ${file}]` : "[An attachment SLICC 6 did not export]"
+      }
+    ];
+  });
+  if (message.role === "user") return { role: "user", content: texts, timestamp };
+  const id = message.toolCallId;
+  if (message.role === "tool-result")
+    return {
+      role: "toolResult",
+      toolCallId: id,
+      toolName: names.get(id) ?? "tool",
+      content: texts,
+      isError: message.isError === true,
+      timestamp
+    };
+  const calls = message.content.flatMap((part) => {
+    if (part.type !== "tool-call") return [];
+    names.set(part.id, part.name);
+    const input = part.input;
+    const object = typeof input === "object" && input !== null && !Array.isArray(input);
+    return [
+      {
+        type: "toolCall",
+        id: part.id,
+        name: part.name,
+        arguments: object ? input : { input }
+      }
+    ];
+  });
+  const assistant = {
+    role: "assistant",
+    content: [...texts, ...calls],
+    api: message.model?.api ?? "slicc-6",
+    provider: message.model?.provider ?? "slicc-6",
+    model: message.model?.id ?? "unknown",
+    usage: message.usage ?? ZERO,
+    stopReason: calls.length ? "toolUse" : "stop",
+    timestamp
+  };
+  return assistant;
+}
+function entriesOf(conversation, files) {
+  const names = /* @__PURE__ */ new Map();
+  return (conversation?.messages ?? []).map((message) => ({
+    kind: V6_KIND,
+    data: { id: message.id },
+    model: [piMessage(message, names, files)]
+  }));
+}
+async function conversationIn(tx, conversation, unit, model) {
+  const record = await tx.createConversation({ ownership: { kind: "ownerless" } });
+  if (model) await configure(tx, record.id, { model });
+  for (const entry of entriesOf(conversation, unit.files)) await tx.appendEntry(record.id, entry);
+  return record.id;
+}
+function modelOf(conversation) {
+  const model = conversation?.messages.findLast((message) => message.model)?.model;
+  return model ? `${model.provider}/${model.id}` : "";
+}
+async function commit(deps, opened, units, at, context) {
+  const model = (await deps.agents.cone().agent(context)).model;
+  const origin = opened.manifest.producer.origin;
+  const last = TITLE_TURNS.at(-1);
+  return deps.agents.update(async (tx, doc) => {
+    const made = [];
+    for (const unit of units.filter((item) => item.record)) {
+      const frozenId = `frozen-${doc.frozenNext ?? 1}`;
+      doc.frozenNext = (doc.frozenNext ?? 1) + 1;
+      const scoops = [];
+      for (const scoop of unit.scoops) {
+        const id = scoopId(scoop.folder);
+        if (doc.scoops[id])
+          throw new ImportError(`the scoop folder ${scoop.folder} was taken during the import`);
+        const conversation = await conversationIn(tx, scoop.conversation, unit, model);
+        doc.scoops[id] = {
+          name: scoop.name,
+          folder: scoop.folder,
+          cone: unit.cone,
+          conversation,
+          anchor: 0,
+          role: null,
+          kind: "async",
+          parent: unit.cone,
+          depth: 1,
+          roots: null,
+          created: null,
+          dropped: null,
+          gone: true,
+          frozen: frozenId,
+          title: scoop.name,
+          titled: last
+        };
+        scoops.push(id);
+      }
+      doc.frozen ??= {};
+      doc.frozen[frozenId] = {
+        cone: unit.cone,
+        name: unit.name,
+        conversation: await conversationIn(tx, unit.main, unit, model),
+        title: unit.title,
+        model: modelOf(unit.main),
+        messages: (unit.main?.messages ?? []).filter((message) => message.role !== "tool-result").length,
+        frozenAt: unit.frozenAt,
+        scoops,
+        v6: { origin, key: unit.key, ...unit.suffix ? { suffix: unit.suffix } : {} }
+      };
+      made.push(frozenId);
+    }
+    doc.imports ??= {};
+    doc.imports[opened.key] = {
+      at,
+      origin,
+      cones: Object.fromEntries(units.map((unit) => [unit.key, unit.cone])),
+      tails: Object.fromEntries(units.map((unit) => [unit.key, unit.tail]))
+    };
+    return made;
+  }, context);
+}
+async function storageRoom() {
+  const estimate = await globalThis.navigator?.storage?.estimate?.().catch(() => void 0);
+  return estimate?.quota === void 0 ? void 0 : estimate.quota - (estimate.usage ?? 0);
+}
+async function importOpened(deps, opened, options, context) {
+  const now = (deps.now ?? Date.now)();
+  const date = day(now);
+  const writer = new Writer(deps.env, context, options.signal);
+  const stage = `${IMPORTS_DIR}/${opened.key.replace("#", "-part")}`;
+  const total = opened.manifest.files.reduce((sum, file) => sum + file.size, 0);
+  const room = await (deps.room ?? storageRoom)();
+  if (room !== void 0 && room < 2 * total)
+    throw new ImportError("there is not enough free storage for this import");
+  try {
+    await writer.sh(
+      `mkdir -p ${IMPORTS_DIR} && chmod 700 ${IMPORTS_DIR} && rm -rf ${quote(stage)} && mkdir -p ${quote(stage)}`
+    );
+    const data = await verify(opened, writer, stage, options.show);
+    const parsed = await parse(opened, data);
+    await assign(deps, opened, parsed.units, options.again, date, context);
+    const run = {
+      deps,
+      opened,
+      parsed,
+      writer,
+      stage,
+      date,
+      show: options.show,
+      fresh: /* @__PURE__ */ new Map(),
+      ready: /* @__PURE__ */ new Set()
+    };
+    let made;
+    try {
+      await place(run);
+      made = await commit(deps, opened, parsed.units, now, context);
+    } catch (error) {
+      await writer.rollback();
+      throw error;
+    }
+    const cones = [...new Set(parsed.units.map((unit) => unit.cone))];
+    const providers = opened.manifest.providers;
+    return {
+      done: true,
+      preview: null,
+      cones,
+      report: null,
+      text: `Imported ${plural(made.length, "frozen cone")} from ${opened.manifest.producer.origin}${made.length ? `: ${made.join(", ")}` : ""}.
+Re-enter in Settings: ${providers.join(", ") || "nothing"}.
+`,
+      error: null
+    };
+  } finally {
+    await deps.env.remove(stage, { recursive: true, force: true }, BACKGROUND_CONTEXT);
+  }
+}
+async function sourceOf(deps, source, context) {
+  if ("bytes" in source) return bytesSource(source.bytes);
+  const { env } = deps;
+  const real = await env.canonicalPath(source.path, context);
+  const info = real.ok ? await env.fileInfo(real.value, context) : real;
   if (!info.ok) throw new ImportError(`can't read ${source.path}: ${info.error.message}`);
   if (info.value.size > LIMITS.archive) throw new ImportError("the zip is over 2 GiB");
-  const read = await env.readBinaryFile(source.path, context);
+  const file = await opfsFile(info.value.path, deps.opfs);
+  if (file?.size === info.value.size) return blobSource(file);
+  if (info.value.size > LIMITS.file)
+    throw new ImportError(`${source.path} isn't on seven's own disk; copy the zip to /tmp first`);
+  const read = await env.readBinaryFile(info.value.path, context);
   if (!read.ok) throw new ImportError(`can't read ${source.path}: ${read.error.message}`);
-  return read.value;
+  return bytesSource(read.value);
 }
-async function checkProfile(env, source, context) {
-  const opened = await openProfile(await bytesOf(env, source, context));
-  const preview = await previewOf(opened);
-  await checked(opened, await verify(opened));
-  return preview;
+const IDLE = { phase: "idle", done: 0, total: 0, text: "" };
+function createImporter(deps) {
+  const progress = replicatedState(IDLE);
+  let running;
+  const result = (fields) => ({
+    done: false,
+    preview: null,
+    cones: [],
+    report: null,
+    text: "",
+    error: null,
+    ...fields
+  });
+  return {
+    progress,
+    cancel: () => running?.abort(),
+    async run(source, options, context) {
+      if (running) return result({ error: "an import is already running" });
+      const controller = new AbortController();
+      running = controller;
+      const show = (phase, done, total) => progress.replace(BACKGROUND_CONTEXT, {
+        phase,
+        done,
+        total,
+        text: phase === "checking" ? `Checking ${done}/${total}` : `Importing ${done}/${total}`
+      });
+      try {
+        const opened = await openProfile(await sourceOf(deps, source, context));
+        const preview = await previewOf(opened, deps.agents.state());
+        if (options.preview) {
+          const writer = new Writer(deps.env, context, controller.signal);
+          await parse(opened, await verify(opened, writer, null, show));
+          progress.replace(BACKGROUND_CONTEXT, IDLE);
+          return result({ preview, text: `${preview.text}
+` });
+        }
+        if (preview.already !== null && !options.again)
+          return result({
+            preview,
+            text: `Already imported on ${day(preview.already)}; freezer import --again makes fresh copies.
+`
+          });
+        const done = await importOpened(
+          deps,
+          opened,
+          { again: options.again === true, signal: controller.signal, show },
+          context
+        );
+        progress.replace(BACKGROUND_CONTEXT, { ...progress.value, phase: "done", text: done.text });
+        return { ...done, preview };
+      } catch (error) {
+        const message = error instanceof ImportError ? error.message : `the import failed: ${error.message}`;
+        progress.replace(BACKGROUND_CONTEXT, { phase: "failed", done: 0, total: 0, text: message });
+        return result({ error: message });
+      } finally {
+        running = void 0;
+      }
+    }
+  };
 }
 export {
+  IDLE,
+  IMPORTS_DIR,
   LIMITS,
-  checkProfile,
+  V6_KIND,
+  createImporter,
   day,
+  entriesOf,
   openProfile,
   previewOf,
   textOf,
+  unitsOf,
   whereOf
 };
 //# sourceMappingURL=v6.js.map