create-walle 0.9.21 → 0.9.23

package/template/docs/private-memory-and-pii-policy.md

@@ -0,0 +1,69 @@
+# Private Memory And PII Policy
+
+This repository must not contain personal memory, private user projects, local
+workspace data, real contact information, or provider/account-specific
+credentials. Wall-E memory and CTM session context are runtime data, not source
+artifacts.
+
+## Keep Out Of Git
+
+- Private journals, including `builder-journal.md`.
+- CTM and Wall-E databases, WAL/SHM files, logs, screenshots, and JSONL
+  transcripts.
+- Personal project artifacts generated for one user, including A/B screenshots,
+  review outputs, and generated site drafts.
+- Real names, email addresses, phone numbers, home/work addresses, company names,
+  private gateway hostnames, and local machine paths in source, tests, and docs.
+- OAuth tokens, API keys, Portkey/custom gateway headers, and `.env` files.
+
+## Use Safe Fixtures
+
+Use generic fixture names such as `Test Owner`, `Student Example`, `Jordan
+Example`, `ExampleCorp`, `example.com`, `gateway.example.internal`, and
+`/Users/example/...`.
+
+Do not use a real user's current personal project as a regression fixture. If a
+regression needs that shape, reduce it to a generic minimal fixture and remove
+the original names, paths, generated assets, and project-specific copy.
+
+## Builder Journal
+
+The builder journal is private memory. The canonical local copy lives outside
+this repository:
+
+```text
+~/.codex/memories/builder-journal.md
+```
+
+If a copy appears in the repo, move or merge it into the private location, then
+delete it from the Git worktree before committing.
+
+## Pre-Commit Guard
+
+Run:
+
+```bash
+node scripts/check-private-data.js
+```
+
+The scanner checks tracked files only. It also reads optional local denylist
+files that are intentionally ignored by Git:
+
+```text
+.private-data-denylist
+.private-data-denylist.local
+~/.codex/private-data-denylist
+~/.codex/memories/private-data-denylist
+```
+
+Put user-specific terms in one of those local files, one term per line. Do not
+commit the denylist itself. The built-in scanner is intentionally
+high-confidence to avoid blocking generic fixtures; user-specific names,
+projects, domains, and private organizations belong in the local denylist.
+
+## Git History
+
+A forward cleanup removes the data from the current tree. Purging old commits is
+a separate destructive operation because it rewrites history and changes commit
+IDs. Use a dedicated history-rewrite plan, coordinate with all clones/remotes,
+and validate that no needed private-only material is lost.

package/template/package.json

@@ -1,6 +1,6 @@
 {
   "name": "walle",
-  "version": "0.9.21",
+  "version": "0.9.23",
   "private": true,
   "description": "Wall-E — your personal digital twin",
   "scripts": {
@@ -9,6 +9,7 @@
     "setup": "node bin/setup.js",
     "clean:dev": "node bin/ctm-dev-cleanup.js --kill --stale --max-age-min 1440",
     "doctor:dev": "node bin/ctm-dev-cleanup.js --stale --max-age-min 1440",
+    "scan:private": "node scripts/check-private-data.js",
     "dev": "bash bin/dev.sh",
     "dev:fresh": "bash bin/dev.sh --fresh",
     "dev:refresh": "bash bin/dev.sh --refresh"

package/template/scripts/check-private-data.js

@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+const ROOT = execFileSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' }).trim();
+
+const TEXT_EXTENSIONS = new Set([
+  '.cjs', '.css', '.csv', '.env', '.example', '.html', '.js', '.json', '.jsonl',
+  '.jsx', '.mjs', '.md', '.py', '.rb', '.sh', '.sql', '.svg', '.test', '.toml',
+  '.ts', '.tsx', '.txt', '.xml', '.yaml', '.yml',
+]);
+
+const RESERVED_EMAIL_DOMAINS = new Set([
+  'b.com',
+  'domain.co',
+  'example.com',
+  'example.net',
+  'example.org',
+  'github.com',
+  'new.com',
+  'openai.com',
+  'saved.example.com',
+  'test',
+  'test.com',
+  'test.org',
+  'test.invalid',
+  'vendor.com',
+  'localhost',
+]);
+
+const RESERVED_USER_PATHS = new Set([
+  '...',
+  'alice',
+  'bob',
+  'example',
+  'foo',
+  'me',
+  'someone',
+  'test',
+  'tester',
+  'runner',
+  'u',
+  'user',
+  'x',
+  'you',
+]);
+
+const CONSUMER_EMAIL_DOMAINS = new Set([
+  'gmail.com',
+  'googlemail.com',
+  'hotmail.com',
+  'icloud.com',
+  'live.com',
+  'me.com',
+  'outlook.com',
+  'yahoo.com',
+]);
+
+const BLOCKED_TRACKED_PATHS = [
+  /^builder-journal\.md$/,
+  /(^|\/)builder-journal\.md$/,
+  /^wall-e\/docs\/ab-screenshots\//,
+];
+
+const SECRET_PATTERNS = [
+  { name: 'hardcoded Anthropic key', regex: /\bsk-ant-[A-Za-z0-9_-]{12,}\b/g },
+  { name: 'hardcoded OpenAI key', regex: /\bsk-proj-[A-Za-z0-9_-]{12,}\b|\bsk-[A-Za-z0-9_-]{24,}\b/g },
+  { name: 'Slack token', regex: /\bxox[baprs]-[A-Za-z0-9-]{12,}\b/g },
+  { name: 'GitHub token', regex: /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g },
+  { name: 'private key block', regex: /-----BEGIN [A-Z ]*PRIVATE KEY-----/g },
+  { name: 'JWT-like token', regex: /\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g },
+];
+
+function gitFiles() {
+  const out = execFileSync('git', ['ls-files', '-z'], { cwd: ROOT });
+  return out.toString('utf8').split('\0').filter(Boolean);
+}
+
+function isProbablyText(file) {
+  const ext = path.extname(file).toLowerCase();
+  if (TEXT_EXTENSIONS.has(ext)) return true;
+  if (file.endsWith('LICENSE') || file.endsWith('Makefile') || file.endsWith('.gitignore')) return true;
+  return false;
+}
+
+function readOptionalDenylist(file) {
+  try {
+    return fs.readFileSync(file, 'utf8')
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter((line) => line && !line.startsWith('#'));
+  } catch {
+    return [];
+  }
+}
+
+function loadPrivateTerms() {
+  const files = [
+    path.join(ROOT, '.private-data-denylist'),
+    path.join(ROOT, '.private-data-denylist.local'),
+    path.join(os.homedir(), '.codex', 'private-data-denylist'),
+    path.join(os.homedir(), '.codex', 'memories', 'private-data-denylist'),
+  ];
+  return files.flatMap(readOptionalDenylist);
+}
+
+function lineColumn(text, index) {
+  const before = text.slice(0, index);
+  const lines = before.split(/\n/);
+  return { line: lines.length, column: lines[lines.length - 1].length + 1 };
+}
+
+function addFinding(findings, file, reason, text, index = 0) {
+  const loc = lineColumn(text, index);
+  findings.push({ file, line: loc.line, column: loc.column, reason });
+}
+
+function scanEmails(findings, file, text) {
+  const emailRe = /\b[A-Z0-9._%+-]+@([A-Z0-9.-]+\.[A-Z]{2,}|localhost|test)\b/gi;
+  for (const match of text.matchAll(emailRe)) {
+    const domain = match[1].toLowerCase();
+    if (RESERVED_EMAIL_DOMAINS.has(domain)) continue;
+    if (domain.endsWith('.example')) continue;
+    if (domain.endsWith('.example.com') || domain.endsWith('.example.net') || domain.endsWith('.example.org')) continue;
+    if (domain.endsWith('.invalid')) continue;
+    if (CONSUMER_EMAIL_DOMAINS.has(domain)) {
+      addFinding(findings, file, `consumer email address: ${match[0]}`, text, match.index);
+    }
+  }
+}
+
+function scanLocalPaths(findings, file, text) {
+  const pathRe = /\/Users\/([A-Za-z0-9._-]+)(?=\/)/g;
+  for (const match of text.matchAll(pathRe)) {
+    if (RESERVED_USER_PATHS.has(match[1])) continue;
+    addFinding(findings, file, `non-generic local user path: /Users/${match[1]}`, text, match.index);
+  }
+}
+
+function scanSecrets(findings, file, text) {
+  for (const pattern of SECRET_PATTERNS) {
+    for (const match of text.matchAll(pattern.regex)) {
+      const normalized = match[0].toLowerCase();
+      if (match[0] === 'sk-ant-...') continue;
+      if (/fake|test|unused|example|dummy|placeholder|redacted|bearer|from|token|key|stale|rotated/.test(normalized)) continue;
+      if (/abc|123/.test(normalized)) continue;
+      addFinding(findings, file, pattern.name, text, match.index);
+    }
+  }
+}
+
+function scanPrivateTerms(findings, file, text, terms) {
+  for (const term of terms) {
+    const idx = text.toLowerCase().indexOf(term.toLowerCase());
+    if (idx !== -1) addFinding(findings, file, `local private denylist term: ${term}`, text, idx);
+  }
+}
+
+function main() {
+  const files = gitFiles();
+  const privateTerms = loadPrivateTerms();
+  const findings = [];
+
+  for (const file of files) {
+    for (const pattern of BLOCKED_TRACKED_PATHS) {
+      if (pattern.test(file)) {
+        findings.push({ file, line: 1, column: 1, reason: 'blocked private/generated path is tracked' });
+      }
+    }
+
+    if (!isProbablyText(file)) continue;
+
+    let text = '';
+    try {
+      text = fs.readFileSync(path.join(ROOT, file), 'utf8');
+    } catch {
+      continue;
+    }
+
+    scanEmails(findings, file, text);
+    scanLocalPaths(findings, file, text);
+    scanSecrets(findings, file, text);
+    scanPrivateTerms(findings, file, text, privateTerms);
+  }
+
+  if (findings.length) {
+    console.error('Private-data scan failed:');
+    for (const finding of findings) {
+      console.error(`- ${finding.file}:${finding.line}:${finding.column} ${finding.reason}`);
+    }
+    process.exit(1);
+  }
+
+  console.log(`Private-data scan passed (${files.length} tracked files checked).`);
+}
+
+main();

package/template/shared/sqlite-owner-guard.js

@@ -0,0 +1,30 @@
+'use strict';
+
+function _truthy(value) {
+  return /^(1|true|yes|on)$/i.test(String(value || '').trim());
+}
+
+function assertNoForeignSqliteOwner({
+  blockedEnv,
+  overrideEnv,
+  databaseLabel,
+  ownerLabel,
+} = {}) {
+  const blockedRole = blockedEnv ? process.env[blockedEnv] : '';
+  if (!blockedRole || (overrideEnv && _truthy(process.env[overrideEnv]))) return;
+
+  const err = new Error(
+    `${databaseLabel || 'SQLite database'} must be accessed through its ${ownerLabel || 'owner'} API; ` +
+    `refusing direct access inside process role ${blockedEnv}=${blockedRole}.` +
+    (overrideEnv ? ` Set ${overrideEnv}=1 only for explicit standalone repair.` : '')
+  );
+  err.code = 'SQLITE_OWNER_VIOLATION';
+  err.blockedEnv = blockedEnv;
+  err.blockedRole = blockedRole;
+  err.overrideEnv = overrideEnv || '';
+  throw err;
+}
+
+module.exports = {
+  assertNoForeignSqliteOwner,
+};

package/template/shared/sqlite-owner-write-queue.js

@@ -0,0 +1,225 @@
+'use strict';
+
+const DEFAULT_MAX_DEPTH = 1000;
+const DEFAULT_DRAIN_TIMEOUT_MS = 5000;
+
+function _durationMs(value, fallback) {
+  const n = Number(value);
+  return Number.isFinite(n) ? Math.max(0, Math.trunc(n)) : fallback;
+}
+
+function _queueError(code, message, extra = {}) {
+  const err = new Error(message);
+  err.code = code;
+  for (const [key, value] of Object.entries(extra)) err[key] = value;
+  return err;
+}
+
+class SqliteOwnerWriteQueue {
+  constructor(options = {}) {
+    this.name = options.name || options.label || 'sqlite-owner-write-queue';
+    this.maxDepth = Math.max(1, Math.trunc(Number(options.maxDepth) || DEFAULT_MAX_DEPTH));
+    this._queue = [];
+    this._running = false;
+    this._active = null;
+    this._accepting = true;
+    this._closed = false;
+    this._enqueued = 0;
+    this._completed = 0;
+    this._failed = 0;
+    this._idleWaiters = [];
+  }
+
+  enqueue(fn, options = {}) {
+    if (typeof fn !== 'function') {
+      throw new TypeError('SqliteOwnerWriteQueue.enqueue requires a function');
+    }
+    if (!this._accepting || this._closed) {
+      return Promise.reject(_queueError(
+        'SQLITE_OWNER_WRITE_QUEUE_CLOSED',
+        `${this.name} is closed`
+      ));
+    }
+    if (this._queue.length >= this.maxDepth) {
+      return Promise.reject(_queueError(
+        'SQLITE_OWNER_WRITE_QUEUE_FULL',
+        `${this.name} is full`,
+        { pending: this._queue.length, maxDepth: this.maxDepth }
+      ));
+    }
+
+    const signal = options.signal || null;
+    if (signal?.aborted) {
+      return Promise.reject(_queueError(
+        'SQLITE_OWNER_WRITE_QUEUE_ABORTED',
+        `${this.name} task aborted before enqueue`
+      ));
+    }
+
+    const label = options.label || 'write';
+    let task;
+    const promise = new Promise((resolve, reject) => {
+      task = {
+        fn,
+        label,
+        resolve,
+        reject,
+        enqueuedAt: Date.now(),
+        started: false,
+        signal,
+        abortListener: null,
+      };
+    });
+
+    if (signal) {
+      task.abortListener = () => {
+        if (task.started) return;
+        const idx = this._queue.indexOf(task);
+        if (idx >= 0) this._queue.splice(idx, 1);
+        task.reject(_queueError(
+          'SQLITE_OWNER_WRITE_QUEUE_ABORTED',
+          `${this.name} task aborted while queued`,
+          { label }
+        ));
+        this._notifyIdleIfNeeded();
+      };
+      signal.addEventListener('abort', task.abortListener, { once: true });
+    }
+
+    this._enqueued += 1;
+    this._queue.push(task);
+    this._pump();
+    return promise;
+  }
+
+  getStatus() {
+    return {
+      name: this.name,
+      active: this._active,
+      running: this._running,
+      pending: this._queue.length,
+      accepting: this._accepting,
+      closed: this._closed,
+      maxDepth: this.maxDepth,
+      enqueued: this._enqueued,
+      completed: this._completed,
+      failed: this._failed,
+      idle: !this._running && this._queue.length === 0,
+    };
+  }
+
+  async drain(options = {}) {
+    const timeoutMs = _durationMs(options.timeoutMs, DEFAULT_DRAIN_TIMEOUT_MS);
+    if (!this._running && this._queue.length === 0) {
+      return { ok: true, timedOut: false, ...this.getStatus() };
+    }
+
+    if (timeoutMs === 0) {
+      await new Promise((resolve) => this._idleWaiters.push(resolve));
+      return { ok: true, timedOut: false, ...this.getStatus() };
+    }
+
+    let timer = null;
+    let idleWaiter = null;
+    try {
+      return await Promise.race([
+        new Promise((resolve) => {
+          idleWaiter = () => resolve({
+            ok: true,
+            timedOut: false,
+            ...this.getStatus(),
+          });
+          this._idleWaiters.push(idleWaiter);
+        }),
+        new Promise((resolve) => {
+          timer = setTimeout(() => resolve({
+            ok: false,
+            timedOut: true,
+            ...this.getStatus(),
+          }), timeoutMs);
+          if (typeof timer.unref === 'function') timer.unref();
+        }),
+      ]);
+    } finally {
+      if (timer) clearTimeout(timer);
+      if (idleWaiter) {
+        const idx = this._idleWaiters.indexOf(idleWaiter);
+        if (idx >= 0) this._idleWaiters.splice(idx, 1);
+      }
+    }
+  }
+
+  async close(options = {}) {
+    const drain = !!options.drain;
+    this._accepting = false;
+    this._closed = true;
+    if (!drain) {
+      this._rejectPending(_queueError(
+        'SQLITE_OWNER_WRITE_QUEUE_CLOSED',
+        `${this.name} closed with pending writes rejected`
+      ));
+      return { ok: true, drained: false, ...this.getStatus() };
+    }
+    const result = await this.drain({ timeoutMs: options.timeoutMs });
+    return { ...result, drained: !result.timedOut };
+  }
+
+  _pump() {
+    if (this._running) return;
+    const task = this._queue.shift();
+    if (!task) {
+      this._notifyIdleIfNeeded();
+      return;
+    }
+
+    this._running = true;
+    this._active = task.label;
+    task.started = true;
+    if (task.signal && task.abortListener) {
+      task.signal.removeEventListener('abort', task.abortListener);
+      task.abortListener = null;
+    }
+
+    Promise.resolve()
+      .then(() => task.fn())
+      .then((value) => {
+        this._completed += 1;
+        task.resolve(value);
+      }, (err) => {
+        this._failed += 1;
+        task.reject(err);
+      })
+      .finally(() => {
+        this._running = false;
+        this._active = null;
+        this._pump();
+      });
+  }
+
+  _rejectPending(err) {
+    const pending = this._queue.splice(0);
+    for (const task of pending) {
+      if (task.signal && task.abortListener) {
+        task.signal.removeEventListener('abort', task.abortListener);
+        task.abortListener = null;
+      }
+      task.reject(err);
+    }
+    this._notifyIdleIfNeeded();
+  }
+
+  _notifyIdleIfNeeded() {
+    if (this._running || this._queue.length > 0) return;
+    const waiters = this._idleWaiters.splice(0);
+    for (const resolve of waiters) resolve();
+  }
+}
+
+function createSqliteOwnerWriteQueue(options = {}) {
+  return new SqliteOwnerWriteQueue(options);
+}
+
+module.exports = {
+  SqliteOwnerWriteQueue,
+  createSqliteOwnerWriteQueue,
+};

package/template/shared/sqlite-storage-policy.js

@@ -0,0 +1,111 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const CLOUD_STORAGE_PATTERNS = Object.freeze([
+  { id: 'dropbox', label: 'Dropbox', regex: /(^|[\\/])(?:Dropbox|CloudStorage[\\/][^\\/]*Dropbox[^\\/]*)([\\/]|$)/i },
+  { id: 'icloud', label: 'iCloud Drive', regex: /(^|[\\/])(?:Mobile Documents|CloudStorage[\\/][^\\/]*iCloud[^\\/]*)([\\/]|$)/i },
+  { id: 'onedrive', label: 'OneDrive', regex: /(^|[\\/])(?:OneDrive|CloudStorage[\\/][^\\/]*OneDrive[^\\/]*)([\\/]|$)/i },
+  { id: 'google-drive', label: 'Google Drive', regex: /(^|[\\/])(?:Google Drive|CloudStorage[\\/][^\\/]*GoogleDrive[^\\/]*)([\\/]|$)/i },
+  { id: 'box', label: 'Box Drive', regex: /(^|[\\/])(?:Box|CloudStorage[\\/][^\\/]*Box[^\\/]*)([\\/]|$)/i },
+]);
+
+const VALID_POLICY_MODES = new Set(['off', 'warn', 'error']);
+const warnedKeys = new Set();
+
+function _normalizePath(value) {
+  const raw = String(value || '').trim();
+  if (!raw) return '';
+  try {
+    const dir = path.dirname(raw);
+    const base = path.basename(raw);
+    const realDir = fs.existsSync(dir) ? fs.realpathSync(dir) : path.resolve(dir);
+    return path.join(realDir, base);
+  } catch {
+    return path.resolve(raw);
+  }
+}
+
+function _truthy(value) {
+  return /^(1|true|yes|on)$/i.test(String(value || '').trim());
+}
+
+function sqliteStoragePolicyMode({ env = process.env, prefix = '' } = {}) {
+  const normalizedPrefix = String(prefix || '').trim().replace(/[^A-Z0-9_]/gi, '_').toUpperCase();
+  if (normalizedPrefix && _truthy(env[`${normalizedPrefix}_ALLOW_UNSAFE_CLOUD_SQLITE`])) return 'off';
+  if (_truthy(env.ALLOW_UNSAFE_CLOUD_SQLITE)) return 'off';
+
+  const raw = String(
+    (normalizedPrefix && env[`${normalizedPrefix}_SQLITE_STORAGE_POLICY`])
+    || env.SQLITE_STORAGE_POLICY
+    || 'warn'
+  ).trim().toLowerCase();
+  return VALID_POLICY_MODES.has(raw) ? raw : 'warn';
+}
+
+function classifySqlitePathRisk(dbPath) {
+  const normalizedPath = _normalizePath(dbPath);
+  const normalizedForMatch = normalizedPath.replace(/\\/g, '/');
+  const match = CLOUD_STORAGE_PATTERNS.find((candidate) => candidate.regex.test(normalizedForMatch));
+  if (!match) {
+    return {
+      risky: false,
+      reason: '',
+      provider: '',
+      providerLabel: '',
+      dbPath: normalizedPath,
+      message: '',
+    };
+  }
+  return {
+    risky: true,
+    reason: 'cloud_sync_sqlite_wal',
+    provider: match.id,
+    providerLabel: match.label,
+    dbPath: normalizedPath,
+    message:
+      `${match.label} sync storage is unsafe for a live SQLite WAL database. `
+      + 'Move the live DB directory to a local disk path and sync backups/exports instead.',
+  };
+}
+
+function enforceSqliteStoragePolicy(dbPath, {
+  env = process.env,
+  prefix = '',
+  label = 'SQLite database',
+  logger = console,
+} = {}) {
+  const risk = classifySqlitePathRisk(dbPath);
+  const mode = sqliteStoragePolicyMode({ env, prefix });
+  const result = { ...risk, mode, blocked: false, label };
+  if (!risk.risky || mode === 'off') return result;
+
+  const message = `${label}: ${risk.message} Path: ${risk.dbPath}`;
+  if (mode === 'error') {
+    const err = new Error(message);
+    err.code = 'SQLITE_UNSAFE_CLOUD_STORAGE';
+    err.storageRisk = result;
+    throw err;
+  }
+
+  const warnKey = `${label}:${risk.provider}:${risk.dbPath}`;
+  if (!warnedKeys.has(warnKey)) {
+    warnedKeys.add(warnKey);
+    try {
+      logger.warn(`[sqlite-storage] ${message}`);
+    } catch {}
+  }
+  return result;
+}
+
+function resetSqliteStoragePolicyWarnings() {
+  warnedKeys.clear();
+}
+
+module.exports = {
+  classifySqlitePathRisk,
+  enforceSqliteStoragePolicy,
+  resetSqliteStoragePolicyWarnings,
+  sqliteStoragePolicyMode,
+};