openmates 0.15.0-alpha.36 → 0.15.0-alpha.38

package/dist/{chunk-BEKDFMUN.js → chunk-N2TP5WZI.js}

@@ -1739,13 +1739,6 @@ var OpenMatesWsClient = class {
           });
           return;
         }
-        if (options?.recoveryTurnId && recoveryJobId && aiResponseDone && !postProcessingDone) {
-          if (postProcessingTimer) {
-            clearTimeout(postProcessingTimer);
-            postProcessingTimer = null;
-          }
-          postProcessingDone = true;
-        }
         if (!aiResponseDone || !postProcessingDone) return;
         if (options?.recoveryTurnId && !recoveryJobId) return;
         if (pendingSubChatHandlers.size > 0) return;
@@ -21795,6 +21788,243 @@ function redactUrl(rawUrl) {
   return url.toString();
 }
 
+// src/accountExportArchive.ts
+var ACCOUNT_EXPORT_FORBIDDEN_FIELD_NAMES = /* @__PURE__ */ new Set([
+  "access_token",
+  "api_key",
+  "backup_code_hash",
+  "encrypted_master_key",
+  "lookup_hash",
+  "password_hash",
+  "private_key",
+  "raw_key",
+  "refresh_token",
+  "share_key",
+  "signing_secret",
+  "token_hash",
+  "totp_seed",
+  "webhook_secret"
+]);
+var ACCOUNT_EXPORT_REDACTION_CATEGORIES = [
+  "api_credentials",
+  "authentication_tokens",
+  "key_material",
+  "password_and_recovery_hashes",
+  "webhook_secrets"
+];
+var ACCOUNT_EXPORT_FORBIDDEN_VALUE_PATTERNS = [
+  /-----BEGIN [A-Z ]*PRIVATE KEY-----/,
+  /(?:^|[^a-z0-9])sk-(?:api|proj|live|test)[-_a-z0-9]{6,}/i,
+  /#key=[A-Za-z0-9_-]{8,}/
+];
+async function writeAccountExportArchive(bundle, flags = {}) {
+  const { mkdir: mkdir2, mkdtemp, rm, writeFile: writeFile2 } = await import("fs/promises");
+  const { tmpdir: tmpdir2 } = await import("os");
+  const { execFileSync: execFileSync3 } = await import("child_process");
+  const { join: join7, dirname: dirname6 } = await import("path");
+  const exportId = safeArchiveSegment(String(bundle.export.export_id ?? `export-${Date.now()}`));
+  const archiveFormat = flags.format === "directory" ? "directory" : "zip";
+  const outputPath = typeof flags.output === "string" ? flags.output : join7(process.cwd(), archiveFormat === "zip" ? `openmates-account-export-${exportId}.zip` : `openmates-account-export-${exportId}`);
+  const stagingDir = archiveFormat === "directory" ? outputPath : await mkdtemp(join7(tmpdir2(), `openmates-account-export-${exportId}-`));
+  const writtenFiles = [];
+  async function writeArchiveText(relativePath, content) {
+    assertAccountExportTextSafe(content, relativePath);
+    const fullPath = join7(stagingDir, relativePath);
+    await mkdir2(dirname6(fullPath), { recursive: true });
+    await writeFile2(fullPath, content, "utf-8");
+    writtenFiles.push(relativePath);
+  }
+  await writeArchiveText("README.md", buildAccountExportReadme(bundle));
+  await writeArchiveText("manifest.yml", serializeArchiveYaml(bundle.manifest));
+  await writeArchiveText("manifest.json", `${JSON.stringify(bundle.manifest, null, 2)}
+`);
+  await writeArchiveText("export-report.yml", serializeArchiveYaml(buildAccountExportReport(bundle)));
+  const domainPayloads = collectAccountExportDomainPayloads(bundle.chunks);
+  for (const [domain, payloads] of Object.entries(domainPayloads)) {
+    const domainFile = payloads.length === 1 ? payloads[0] : { chunks: payloads };
+    await writeArchiveText(`domains/${safeArchiveSegment(domain)}.json`, `${JSON.stringify(domainFile, null, 2)}
+`);
+  }
+  await writeAccountExportChatFiles(domainPayloads.chats ?? [], writeArchiveText);
+  if (archiveFormat === "directory") return { output: outputPath, format: "directory", files: writtenFiles.length };
+  await mkdir2(dirname6(outputPath), { recursive: true });
+  try {
+    execFileSync3("zip", ["-r", outputPath, "."], { cwd: stagingDir, stdio: "pipe" });
+    return { output: outputPath, format: "zip", files: writtenFiles.length };
+  } finally {
+    await rm(stagingDir, { recursive: true, force: true });
+  }
+}
+function assertAccountExportPayloadSafe(value, path = "$export") {
+  if (value === null || value === void 0) return;
+  if (typeof value === "string") {
+    for (const pattern of ACCOUNT_EXPORT_FORBIDDEN_VALUE_PATTERNS) {
+      if (pattern.test(value)) throw new Error(`Account export contains forbidden secret-like value at ${path}`);
+    }
+    return;
+  }
+  if (typeof value !== "object") return;
+  if (Array.isArray(value)) {
+    value.forEach((item, index) => assertAccountExportPayloadSafe(item, `${path}[${index}]`));
+    return;
+  }
+  for (const [key, child] of Object.entries(value)) {
+    const normalizedKey = key.toLowerCase();
+    if (ACCOUNT_EXPORT_FORBIDDEN_FIELD_NAMES.has(normalizedKey)) {
+      throw new Error(`Account export contains forbidden secret field '${key}' at ${path}`);
+    }
+    assertAccountExportPayloadSafe(child, `${path}.${key}`);
+  }
+}
+function sanitizeAccountExportManifest(manifest) {
+  const copy = JSON.parse(JSON.stringify(manifest));
+  const report = copy.report;
+  if (report && typeof report === "object" && !Array.isArray(report)) {
+    const reportObject = report;
+    if (Array.isArray(reportObject.redactions)) {
+      reportObject.redactions = ACCOUNT_EXPORT_REDACTION_CATEGORIES;
+    }
+  }
+  assertAccountExportPayloadSafe(copy, "$manifest");
+  return copy;
+}
+function assertAccountExportTextSafe(content, relativePath) {
+  for (const pattern of ACCOUNT_EXPORT_FORBIDDEN_VALUE_PATTERNS) {
+    if (pattern.test(content)) throw new Error(`Account export file ${relativePath} contains forbidden secret-like content`);
+  }
+  for (const field of ACCOUNT_EXPORT_FORBIDDEN_FIELD_NAMES) {
+    if (new RegExp(`"?${field}"?\\s*:`, "i").test(content)) {
+      throw new Error(`Account export file ${relativePath} contains forbidden secret field '${field}'`);
+    }
+  }
+}
+function buildAccountExportReadme(bundle) {
+  const selectedDomains = Array.isArray(bundle.manifest.selected_domains) ? bundle.manifest.selected_domains.map(String) : [];
+  return [
+    "# OpenMates Account Export",
+    "",
+    `Export ID: ${String(bundle.export.export_id ?? "unknown")}`,
+    `Status: ${String(bundle.export.status ?? "unknown")}`,
+    `Schema: ${String(bundle.manifest.schema_version ?? "account-export-v1")}`,
+    `Exported at: ${(/* @__PURE__ */ new Date()).toISOString()}`,
+    "",
+    "## Contents",
+    "",
+    "- `manifest.yml` records selected domains, filters, and domain status.",
+    "- `export-report.yml` summarizes completion, failures, and redacted categories.",
+    "- `domains/` contains one JSON file per exported domain.",
+    "- `chats/` contains one Markdown and one YAML file per chat when chat rows include readable content.",
+    "",
+    "## Selected Domains",
+    "",
+    ...selectedDomains.length > 0 ? selectedDomains.map((domain) => `- ${domain}`) : ["- none"],
+    "",
+    "Reusable credentials, token hashes, private keys, and raw secrets are intentionally excluded.",
+    ""
+  ].join("\n");
+}
+function buildAccountExportReport(bundle) {
+  const report = bundle.manifest.report && typeof bundle.manifest.report === "object" && !Array.isArray(bundle.manifest.report) ? bundle.manifest.report : {};
+  return {
+    export_id: bundle.export.export_id ?? null,
+    status: bundle.export.status ?? report.status ?? "unknown",
+    selected_domains: Array.isArray(bundle.manifest.selected_domains) ? bundle.manifest.selected_domains : [],
+    failures: Array.isArray(report.failures) ? report.failures : [],
+    redacted_secret_categories: ACCOUNT_EXPORT_REDACTION_CATEGORIES,
+    partial_requires_acceptance: Boolean(report.partial_requires_acceptance)
+  };
+}
+function collectAccountExportDomainPayloads(chunks) {
+  const grouped = {};
+  for (const chunk of chunks) {
+    const domain = safeArchiveSegment(String(chunk.domain ?? "unknown"));
+    const payload = chunk.payload && typeof chunk.payload === "object" && !Array.isArray(chunk.payload) ? chunk.payload : { value: chunk.payload ?? null };
+    grouped[domain] = grouped[domain] ?? [];
+    grouped[domain].push({
+      chunk_id: chunk.chunk_id ?? null,
+      sequence: chunk.sequence ?? null,
+      status: chunk.status ?? null,
+      ...payload
+    });
+  }
+  return grouped;
+}
+async function writeAccountExportChatFiles(chatPayloads, writeArchiveText) {
+  const chats = [];
+  for (const payload of chatPayloads) {
+    if (Array.isArray(payload.items)) {
+      chats.push(...payload.items.filter((item) => item !== null && typeof item === "object" && !Array.isArray(item)));
+    }
+  }
+  for (const [index, chat] of chats.entries()) {
+    const chatId = safeArchiveSegment(String(chat.id ?? chat.chat_id ?? `chat-${index + 1}`));
+    await writeArchiveText(`chats/${chatId}.yml`, serializeArchiveYaml({ chat, messages: Array.isArray(chat.messages) ? chat.messages : [] }));
+    await writeArchiveText(`chats/${chatId}.md`, buildAccountExportChatMarkdown(chat));
+  }
+}
+function buildAccountExportChatMarkdown(chat) {
+  const title = String(chat.title ?? chat.name ?? chat.id ?? chat.chat_id ?? "Untitled Chat");
+  const lines = [`# ${title}`, ""];
+  if (typeof chat.summary === "string" && chat.summary.trim()) {
+    lines.push("## Summary", "", chat.summary.trim(), "");
+  }
+  const messages = Array.isArray(chat.messages) ? chat.messages : [];
+  if (messages.length > 0) {
+    lines.push("## Messages", "");
+    for (const message of messages) {
+      if (!message || typeof message !== "object" || Array.isArray(message)) continue;
+      const record = message;
+      lines.push(`### ${String(record.role ?? record.sender ?? "message")}`, "", String(record.content ?? record.text ?? ""), "");
+    }
+  } else {
+    lines.push("No readable message records were included in this export chunk.", "");
+  }
+  return `${lines.join("\n")}
+`;
+}
+function serializeArchiveYaml(data, indent = 0) {
+  const pad2 = "  ".repeat(indent);
+  let out = "";
+  for (const [key, val] of Object.entries(data)) {
+    if (val === null || val === void 0) {
+      out += `${pad2}${key}: null
+`;
+    } else if (typeof val === "boolean" || typeof val === "number") {
+      out += `${pad2}${key}: ${val}
+`;
+    } else if (typeof val === "string") {
+      out += val.includes("\n") ? `${pad2}${key}: |
+${val.split("\n").map((line) => `${pad2}  ${line}`).join("\n")}
+` : `${pad2}${key}: ${quoteYamlString(val)}
+`;
+    } else if (Array.isArray(val)) {
+      out += `${pad2}${key}:
+`;
+      for (const item of val) {
+        if (item && typeof item === "object" && !Array.isArray(item)) {
+          out += `${pad2}- 
+${serializeArchiveYaml(item, indent + 2)}`;
+        } else {
+          out += `${pad2}- ${typeof item === "string" ? quoteYamlString(item) : String(item)}
+`;
+        }
+      }
+    } else if (typeof val === "object") {
+      out += `${pad2}${key}:
+${serializeArchiveYaml(val, indent + 1)}`;
+    }
+  }
+  return out;
+}
+function quoteYamlString(value) {
+  const needsQuote = value.includes(":") || value.includes("#") || value.startsWith("{") || value.startsWith("[") || value === "" || ["true", "false", "null"].includes(value);
+  return needsQuote ? `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"` : value;
+}
+function safeArchiveSegment(value) {
+  const cleaned = value.trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
+  return cleaned || "unknown";
+}
+
 // ../ui/src/demo_chats/data/example_chats/gigantic-airplanes.ts
 var giganticAirplanesChat = {
   chat_id: "example-gigantic-airplanes",
@@ -40092,6 +40322,15 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
     new_chat: {
       text: "New chat"
     },
+    ideabucket_badge: {
+      text: "IdeaBucket"
+    },
+    ideabucket_draft: {
+      text: "IdeaBucket draft"
+    },
+    ideabucket_input_pill: {
+      text: "IdeaBucket for today"
+    },
     request_feature: {
       text: "Request feature"
     },
@@ -62969,26 +63208,44 @@ async function runAccountExport(client, flags) {
     client.getAccountExportManifest(exportId),
     client.listAccountExportChunks(exportId)
   ]);
-  const completed = await client.completeAccountExport(exportId);
-  return {
+  const downloadedChunks = [];
+  try {
+    for (const chunk of chunks.chunks) {
+      const chunkId = String(chunk.chunk_id ?? "");
+      const downloaded = chunkId ? await client.getAccountExportChunk(exportId, chunkId) : chunk;
+      assertAccountExportPayloadSafe(downloaded);
+      downloadedChunks.push(downloaded);
+    }
+  } catch (error) {
+    await client.cancelAccountExport(exportId).catch(() => void 0);
+    throw error;
+  }
+  let completed = await client.completeAccountExport(exportId);
+  const completedStatus = String(completed.export.status ?? "");
+  if (completedStatus === "partial") {
+    if (flags["accept-partial"] !== true) {
+      throw new Error(`Account export ${exportId} is partial. Re-run with --accept-partial to accept it, or inspect/cancel the job first.`);
+    }
+    completed = await client.acceptPartialAccountExport(exportId);
+  }
+  const bundle = {
     export: completed.export,
-    manifest: manifest.manifest,
-    chunks: chunks.chunks
+    manifest: sanitizeAccountExportManifest(manifest.manifest),
+    chunks: downloadedChunks
   };
+  const archive = await writeAccountExportArchive(bundle, flags);
+  return { ...bundle, archive };
 }
 function printAccountExportBundle(bundle, flags) {
-  if (typeof flags.output === "string") {
-    writeFileSync7(flags.output, `${JSON.stringify(bundle, null, 2)}
-`, "utf-8");
-  }
   if (flags.json === true) {
-    printJson2(typeof flags.output === "string" ? { ...bundle, output: flags.output } : bundle);
+    printJson2(bundle);
     return;
   }
   const exportRecord = bundle.export && typeof bundle.export === "object" ? bundle.export : {};
+  const archive = bundle.archive && typeof bundle.archive === "object" ? bundle.archive : {};
   process.stdout.write(`Account export ${String(exportRecord.export_id ?? "")} ${String(exportRecord.status ?? "unknown")}
 `);
-  if (typeof flags.output === "string") process.stdout.write(`Wrote ${flags.output}
+  if (typeof archive.output === "string") process.stdout.write(`Wrote ${archive.output}
 `);
 }
 function printApiKeyList(result, flags) {

package/dist/cli.js

@@ -2,7 +2,7 @@
 import {
   getExtForLang,
   serializeToYaml
-} from "./chunk-BEKDFMUN.js";
+} from "./chunk-N2TP5WZI.js";
 import "./chunk-AXNRPVLE.js";
 export {
   getExtForLang,

package/dist/index.js

@@ -18,7 +18,7 @@ import {
   reconcileAuthoritativeChats,
   renderSupportInfo,
   serializeToYaml
-} from "./chunk-BEKDFMUN.js";
+} from "./chunk-N2TP5WZI.js";
 import "./chunk-AXNRPVLE.js";
 export {
   APP_SKILL_METADATA,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmates",
-  "version": "0.15.0-alpha.36",
+  "version": "0.15.0-alpha.38",
   "description": "OpenMates CLI and SDK",
   "type": "module",
   "main": "dist/index.js",
@@ -30,11 +30,13 @@
     "test:unit:workflows": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/workflows.test.ts",
     "test:unit:sdk-workflows": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/sdk-workflows.test.ts",
     "test:unit:remote-access": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/remoteSources.test.ts tests/remoteAccess.test.ts",
+    "test:unit:teams": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/teams-permissions.test.ts",
     "test:unit:proton-bridge-connector": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/protonBridgeConnector.test.ts",
     "test:unit:proton-bridge-connector-integration": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/protonBridgeConnector.integration.test.ts",
     "test:unit:sdk-projects": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/sdk-projects.test.ts",
     "test:unit:billing": "node --test tests/billing.test.ts",
     "test:unit:account-delete": "node --test tests/account-delete.test.ts",
+    "test:unit:ideabucket": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/ideabucket.test.ts",
     "test:unit:cli": "node --test tests/cli.test.ts",
     "test:t-cli-recovery": "node --test --experimental-strip-types --loader ./tests/loader.mjs --test-name-pattern recovery tests/client.test.ts",
     "test:t-cli-update-required": "npm run build && node --test --test-name-pattern update-required tests/cli.test.ts",
@@ -43,7 +45,7 @@
     "test:unit:tui": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/tui.test.ts tests/tuiExampleContinuation.test.ts tests/tuiWorkflowInteraction.test.ts",
     "test:unit:connected-account-import": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/connectedAccountImport.test.ts",
     "test:unit:benchmark": "node --test --experimental-strip-types --loader ./tests/loader.mjs src/__tests__/benchmark.test.ts",
-    "test": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/crypto.test.ts tests/storage.test.ts tests/keychain.test.ts tests/mentions.test.ts tests/outputRedactor.test.ts tests/fileEmbed.test.ts tests/embedCreator.test.ts tests/shareEncryption.test.ts tests/server.test.ts tests/ws.test.ts tests/urlEmbed.test.ts tests/tui.test.ts tests/tuiExampleContinuation.test.ts tests/tuiWorkflowInteraction.test.ts tests/workflows.test.ts tests/sdk-workflows.test.ts tests/draft-sync.test.ts tests/sdk-draft-sync.test.ts src/__tests__/benchmark.test.ts && node --test --experimental-strip-types --loader ./tests/loader.mjs tests/embedRenderers.test.ts tests/connectedAccountImport.test.ts && node --test tests/cli.test.ts tests/billing.test.ts tests/account-delete.test.ts tests/signup.test.ts tests/security-setup.test.ts tests/e2e-provisioning.test.ts"
+    "test": "node --test --experimental-strip-types --loader ./tests/loader.mjs tests/crypto.test.ts tests/storage.test.ts tests/keychain.test.ts tests/mentions.test.ts tests/outputRedactor.test.ts tests/fileEmbed.test.ts tests/embedCreator.test.ts tests/shareEncryption.test.ts tests/server.test.ts tests/ws.test.ts tests/urlEmbed.test.ts tests/tui.test.ts tests/tuiExampleContinuation.test.ts tests/tuiWorkflowInteraction.test.ts tests/workflows.test.ts tests/sdk-workflows.test.ts tests/draft-sync.test.ts tests/sdk-draft-sync.test.ts tests/ideabucket.test.ts tests/teams-permissions.test.ts src/__tests__/benchmark.test.ts && node --test --experimental-strip-types --loader ./tests/loader.mjs tests/embedRenderers.test.ts tests/connectedAccountImport.test.ts && node --test tests/cli.test.ts tests/billing.test.ts tests/account-delete.test.ts tests/signup.test.ts tests/security-setup.test.ts tests/e2e-provisioning.test.ts"
   },
   "keywords": [
     "openmates",

package/templates/core/docker-compose.selfhost.yml

@@ -185,25 +185,27 @@ services:
     restart: on-failure
 
   task-worker:
-    image: ${OPENMATES_IMAGE_REGISTRY:-ghcr.io/glowingkitty}/openmates-api:${OPENMATES_IMAGE_TAG:-dev}
+    <<: *openmates-worker-base
     container_name: task-worker
-    restart: unless-stopped
     mem_limit: ${TASK_WORKER_MEMORY_LIMIT:-3g}
-    env_file: ../../.env
-    command: python -m celery -A backend.core.api.app.tasks.celery_config worker --loglevel=info --queues=email,user_init,persistence,health_check,server_stats,demo,reminder,push --concurrency=${TASK_WORKER_CONCURRENCY:-3} --max-tasks-per-child=50 --max-memory-per-child=600000 --prefetch-multiplier=1
-    networks: [openmates]
-    depends_on: [api]
+    environment:
+      <<: *openmates-worker-env
+      CELERY_QUEUES: email,user_init,persistence,health_check,server_stats,demo,reminder,push
+      CELERY_AUTOSCALE_MAX: ${TASK_WORKER_CONCURRENCY:-${CELERY_AUTOSCALE_MAX:-3}}
+      CELERY_AUTOSCALE_MIN: ${CELERY_AUTOSCALE_MIN:-1}
+      CELERY_METRICS_PORT: "9101"
+    command: >
+      sh -c "chown -R celeryuser:celeryuser /vault-data && gosu celeryuser python -m celery -A backend.core.api.app.tasks.celery_config worker --loglevel=info --queues=email,user_init,persistence,health_check,server_stats,demo,reminder,push --concurrency=$${CELERY_AUTOSCALE_MAX} --max-tasks-per-child=50 --max-memory-per-child=600000 --prefetch-multiplier=1"
 
   task-scheduler:
-    image: ${OPENMATES_IMAGE_REGISTRY:-ghcr.io/glowingkitty}/openmates-api:${OPENMATES_IMAGE_TAG:-dev}
+    <<: *openmates-worker-base
     container_name: task-scheduler
-    restart: unless-stopped
-    env_file: ../../.env
     command: python -m celery -A backend.core.api.app.tasks.celery_config beat --loglevel=info --schedule=/celerybeat-data/celerybeat-schedule
     volumes:
+      - ../../config:/app_config
+      - vault-setup-data:/vault-data
+      - api-logs:/app/logs
       - celerybeat-schedule-data:/celerybeat-data
-    networks: [openmates]
-    depends_on: [api]
 
   app-ai-worker:
     <<: *openmates-worker-base