@rubytech/create-maxy 1.0.776 → 1.0.777

package/dist/index.js

@@ -1541,11 +1541,11 @@ function setupVncViewer() {
 }
 function setupAccount() {
     log("10", TOTAL, "Setting up...");
-    // Tasks 787 + 788 — both seed-neo4j.sh and embed-backfill.sh hard-exit
-    // without NEO4J_URI. The installer owns the brand-correct URI and password,
-    // so we derive them once and pass to both call sites. Missing password file
-    // is a hard error: ensureNeo4jPassword() ran upstream and would have thrown
-    // already if it couldn't reach the brand's Neo4j.
+    // Task 787 — seed-neo4j.sh hard-exits without NEO4J_URI. The installer
+    // owns the brand-correct URI and password, so we derive them once.
+    // Missing password file is a hard error: ensureNeo4jPassword() ran
+    // upstream and would have thrown already if it couldn't reach the
+    // brand's Neo4j.
     const passwordFile = join(INSTALL_DIR, "platform/config/.neo4j-password");
     if (!existsSync(passwordFile)) {
         throw new Error(`Neo4j password file missing at ${passwordFile} — required by setup step.`);
@@ -1559,40 +1559,6 @@ function setupAccount() {
         logFile(`  [neo4j] passing NEO4J_URI=${neo4jUri} to seed`);
         shell("bash", [seedScript], { cwd: INSTALL_DIR, env: neo4jEnv });
     }
-    // Task 748 — universal embedding coverage backfill. Run after seed so the
-    // entity_search index is in place and any pre-Task-748 nodes (e.g. the
-    // 5096 LinkedIn-imported Persons on existing Pis that bulk-import skipped
-    // embedding for) get a vector populated. Idempotent — instant no-op when
-    // nothing is pending, so re-running on every install is harmless.
-    //
-    // Failure-mode policy: WARN, do not abort. The fulltext index is already
-    // applied above, so BM25 search works end-to-end without embeddings; the
-    // only gap is vector ranking quality on legacy nodes. Aborting the
-    // installer on an Ollama hiccup would block every install for a
-    // strictly-degradable feature. The script's own loud-failure output
-    // tells the operator how to re-run.
-    const backfillScript = join(INSTALL_DIR, "platform/scripts/embed-backfill.sh");
-    if (existsSync(backfillScript)) {
-        const start = Date.now();
-        logFile(`> bash ${backfillScript} (warn-not-abort)`);
-        const result = spawnSync("bash", [backfillScript], {
-            stdio: "inherit",
-            timeout: 30 * 60_000,
-            cwd: INSTALL_DIR,
-            env: neo4jEnv,
-        });
-        const dur = ((Date.now() - start) / 1000).toFixed(1);
-        if (result.status !== 0 || result.signal) {
-            const reason = result.signal ? `signal=${result.signal}` : `exit=${result.status}`;
-            logFile(`  WARN: embed-backfill non-zero (${reason}) after ${dur}s`);
-            console.warn(`\n  WARNING: embed-backfill did not complete (${reason}) — BM25 search works,\n` +
-                `  but vector ranking on legacy nodes will be sparse until you re-run:\n` +
-                `    bash ${backfillScript}\n`);
-        }
-        else {
-            logFile(`  OK embed-backfill in ${dur}s`);
-        }
-    }
 }
 // ---------------------------------------------------------------------------
 // Tunnel script shortcuts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rubytech/create-maxy",
-  "version": "1.0.776",
+  "version": "1.0.777",
   "description": "Install Maxy — AI for Productive People",
   "bin": {
     "create-maxy": "./dist/index.js"

package/payload/platform/scripts/embed-backfill.sh

@@ -1,382 +0,0 @@
-#!/usr/bin/env bash
-# ============================================================
-# embed-backfill.sh — populate embeddings on legacy nodes (Task 748)
-#
-# Walks the Neo4j graph for nodes carrying any registered Maxy label that
-# lack `n.embedding` and have at least one populated text property. For
-# each such node the script builds a text representation from the same
-# property union the fulltext index covers (`name`, `title`, `summary`,
-# `headline`, `body`, `content`, `text`), POSTs it to Ollama's `/api/embed`
-# endpoint, and writes the resulting vector back to the node.
-#
-# Why it exists. Pre-Task-748 bulk-import paths (notably `memory-archive-write`
-# for LinkedIn Connections.csv, ~5096 Persons per import) skipped per-row
-# embedding to keep import latency under five minutes. With Task 748's
-# universal fulltext coverage in place, BM25 catches those nodes immediately
-# but vector ranking is sparse until embeddings exist. This script heals
-# both the legacy backlog and any future bulk-imported population.
-#
-# Idempotent. Re-running picks up exactly where a prior run left off because
-# the gating predicate is `n.embedding IS NULL` — nodes embedded by the
-# previous run are excluded from the next batch query.
-#
-# Loud failure (per feedback_loud_failures.md). Any Ollama HTTP failure or
-# cypher-shell error aborts the script with a non-zero exit and prints a
-# precise re-run instruction. Partial-state-on-abort is safe: nodes whose
-# embedding was committed before the abort stay embedded; the rest fall back
-# into the next run's batch.
-#
-# Concurrent-run safety. flock-guarded — a second concurrent invocation
-# exits immediately with a clear message, no work attempted. Protects
-# against operator double-clicks and against the installer running it
-# while a manual run is in flight.
-#
-# Usage. Stand-alone re-run: `bash platform/scripts/embed-backfill.sh`.
-# Installer-driven: invoked automatically post-`seed-neo4j.sh` on every
-# install (the no-op fast path returns in milliseconds when nothing is
-# pending, so re-running on every install is harmless).
-# ============================================================
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
-
-# NEO4J_URI is hard-required (Task 788). The previous default
-# `bolt://localhost:7687` would silently route the backfill to the wrong Neo4j
-# on any brand-dedicated install, masking the actual configuration error.
-if [ -z "${NEO4J_URI:-}" ]; then
-  echo "Error: NEO4J_URI required (no default — see Task 788)" >&2
-  echo "  Set NEO4J_URI=bolt://localhost:<brand.neo4jPort> before running." >&2
-  exit 1
-fi
-NEO4J_USER="${NEO4J_USER:-neo4j}"
-OLLAMA_URL="${OLLAMA_URL:-http://localhost:11434}"
-EMBED_MODEL="${EMBED_MODEL:-nomic-embed-text}"
-BATCH_SIZE="${EMBED_BACKFILL_BATCH_SIZE:-50}"
-
-# Lock file is brand-scoped via the install directory hash so concurrent
-# Maxy + Real Agent installs (or any two brand installs sharing the device)
-# do not block each other unnecessarily — they target separate Neo4j
-# instances under separate INSTALL_DIRs and have zero shared state. The
-# explicit env var override stays for operator-driven workflows.
-INSTALL_DIR_HASH="$(echo -n "$PROJECT_DIR" | shasum | cut -c1-12)"
-LOCK_FILE="${EMBED_BACKFILL_LOCK_FILE:-/tmp/maxy-embed-backfill-${INSTALL_DIR_HASH}.lock}"
-
-# Resolve Neo4j password the same way seed-neo4j.sh does. Explicit env var
-# takes precedence so the installer can pass it through without writing the
-# file twice.
-NEO4J_PASSWORD_FILE="$PROJECT_DIR/config/.neo4j-password"
-if [ -z "${NEO4J_PASSWORD:-}" ]; then
-  if [ -f "$NEO4J_PASSWORD_FILE" ]; then
-    NEO4J_PASSWORD=$(cat "$NEO4J_PASSWORD_FILE")
-  else
-    echo "[embed-backfill] FAILED: NEO4J_PASSWORD env var unset and $NEO4J_PASSWORD_FILE missing"
-    echo "[embed-backfill] re-run after the seed step writes the password file, or set NEO4J_PASSWORD explicitly"
-    exit 1
-  fi
-fi
-export NEO4J_URI NEO4J_USER NEO4J_PASSWORD OLLAMA_URL EMBED_MODEL BATCH_SIZE
-
-if ! command -v cypher-shell >/dev/null 2>&1; then
-  echo "[embed-backfill] FAILED: cypher-shell not on PATH; install Neo4j or add cypher-shell to PATH"
-  exit 1
-fi
-if ! command -v python3 >/dev/null 2>&1; then
-  echo "[embed-backfill] FAILED: python3 not on PATH; the installer requires it"
-  exit 1
-fi
-
-# flock guard — second concurrent invocation exits cleanly. The exec on
-# fd 200 keeps the lock held for the lifetime of this process; flock -n
-# is non-blocking so a busy lock returns immediately rather than queueing.
-exec 200>"$LOCK_FILE"
-if ! flock -n 200; then
-  echo "[embed-backfill] another instance is already running (lock=$LOCK_FILE), skipping"
-  exit 0
-fi
-
-# The python heredoc owns the per-batch loop. It uses subprocess to call
-# cypher-shell (avoids re-implementing Bolt) and urllib to call Ollama
-# (no extra deps). cypher-shell `--format plain` returns CSV; the csv
-# module handles quoting/escaping reliably so node text containing commas,
-# quotes, or newlines round-trips correctly.
-#
-# Cypher contract:
-#   READ:  one row per unembedded node — { id: elementId, text: coalesced }
-#          gated by `n.embedding IS NULL` AND `any(label IN labels(n)
-#          WHERE label IN $registered)` AND a non-empty coalesce of the
-#          text property union. Nodes carrying an :Trashed label are
-#          excluded explicitly. READ params (`registered` list of strings,
-#          `batchSize` int) are passed via cypher-shell `--param` as plain
-#          Cypher expressions (string list literals + integer literal).
-#   WRITE: one batched UNWIND per chunk — pairs of (id, embedding[])
-#          interpolated into the Cypher payload as bare-key map literals
-#          (`{id: '...', embedding: [...]}`). Cypher does NOT accept
-#          double-quoted-string map keys, so JSON-serialised values cannot
-#          be passed via `--param` for the WRITE side; the inline literal
-#          path is the apoc-free alternative.
-#
-# The script does NOT shell out to the existing TS embed() helper because
-# that would require booting Node + the platform/lib build. Calling the
-# Ollama HTTP endpoint directly preserves the same behaviour with zero
-# build dependency.
-exec python3 - <<'PYEOF'
-import json
-import os
-import sys
-import time
-import urllib.error
-import urllib.request
-from subprocess import PIPE, Popen
-from io import StringIO
-import csv
-
-NEO4J_URI = os.environ["NEO4J_URI"]
-NEO4J_USER = os.environ["NEO4J_USER"]
-NEO4J_PASSWORD = os.environ["NEO4J_PASSWORD"]
-OLLAMA_URL = os.environ["OLLAMA_URL"]
-EMBED_MODEL = os.environ["EMBED_MODEL"]
-BATCH_SIZE = int(os.environ["BATCH_SIZE"])
-
-# Mirrors the FOR (n:...) clause of `entity_search` in schema.cypher.
-# Doctrine: every label written by the platform is searchable AND embeddable.
-# Future label additions must extend BOTH this list and schema.cypher; the
-# fulltext-coverage doctrine test catches the schema half but not this list.
-REGISTERED_LABELS = [
-    "LocalBusiness", "Service", "PriceSpecification", "OpeningHoursSpecification", "Organization",
-    "Person", "UserProfile", "Preference", "AdminUser", "AccessGrant",
-    "KnowledgeDocument", "Section", "Chunk", "DigitalDocument", "CreativeWork",
-    "Question", "FAQPage", "DefinedTerm", "Review", "ImageObject",
-    "Conversation", "AdminConversation", "PublicConversation", "Message",
-    "UserMessage", "AssistantMessage", "ToolCall",
-    "Task", "Project", "Event",
-    "Workflow", "WorkflowStep", "WorkflowRun", "StepResult",
-    "OnboardingState", "Email", "EmailAccount", "ReviewAlert",
-    "Position", "Credential",
-]
-
-# Properties to coalesce for the embedding text. Ordered: most identifying
-# property first. Matches the canonical text-property list pinned by the
-# fulltext-coverage doctrine test.
-EMBED_TEXT_PROPS = ["name", "title", "summary", "headline", "body", "content", "text"]
-
-
-def cypher(query: str, params: dict | None = None) -> str:
-    """Run a Cypher statement via cypher-shell --format plain.
-    Returns stdout as a single string. Aborts the script on non-zero exit
-    so a Cypher syntax error or a Neo4j outage surfaces immediately."""
-    cmd = [
-        "cypher-shell", "-u", NEO4J_USER, "-p", NEO4J_PASSWORD, "-a", NEO4J_URI,
-        "--format", "plain",
-    ]
-    if params:
-        for key, value in params.items():
-            cmd.extend(["--param", f"{key} => {json.dumps(value)}"])
-    proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
-    out, err = proc.communicate(query.encode("utf-8"))
-    if proc.returncode != 0:
-        sys.stderr.write(f"[embed-backfill] FAILED: cypher-shell exited {proc.returncode}\n")
-        sys.stderr.write(err.decode("utf-8", errors="replace"))
-        sys.exit(1)
-    return out.decode("utf-8", errors="replace")
-
-
-def parse_csv_rows(stdout: str) -> list[dict]:
-    """cypher-shell --format plain emits a CSV header + rows. The csv module
-    handles quoting reliably even when text contains commas/quotes/newlines.
-
-    skipinitialspace=True is required because cypher-shell emits a space
-    after each comma in both header and data rows (`id, firstLabel, text`),
-    and DictReader otherwise treats the spaces as part of the column name —
-    `row["text"]` raises KeyError because the actual key is " text"."""
-    if not stdout.strip():
-        return []
-    reader = csv.DictReader(StringIO(stdout), skipinitialspace=True)
-    return list(reader)
-
-
-def ollama_embed(text: str, *, timeout: int = 30, retry_on_timeout: bool = True) -> list[float]:
-    """POST text to Ollama /api/embed.
-
-    Cold-start tolerance: when nomic-embed-text is not yet loaded into Ollama's
-    process memory, the first request for the model after a fresh boot can
-    exceed 30s while the model loads. Subsequent requests are fast. We retry
-    ONCE on TimeoutError with a longer (180s) timeout so a cold model load
-    does not abort the entire backfill at the first node. Retry is OFF by
-    default for the warmup probe to avoid recursion.
-
-    Aborts the script (non-zero exit) on any non-recoverable HTTP failure
-    with a precise message + re-run instruction so the operator never thinks
-    the backfill silently completed.
-    """
-    body = json.dumps({"model": EMBED_MODEL, "input": text}).encode("utf-8")
-    req = urllib.request.Request(
-        f"{OLLAMA_URL}/api/embed",
-        data=body,
-        headers={"Content-Type": "application/json"},
-        method="POST",
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            payload = json.loads(resp.read().decode("utf-8"))
-    except TimeoutError as e:
-        if retry_on_timeout:
-            sys.stderr.write(
-                f"[embed-backfill] WARN: Ollama timeout after {timeout}s — likely cold-start; retrying with 180s timeout\n"
-            )
-            return ollama_embed(text, timeout=180, retry_on_timeout=False)
-        sys.stderr.write(f"[embed-backfill] FAILED: Ollama timeout after {timeout}s ({e})\n")
-        sys.stderr.write(
-            f"[embed-backfill] re-run via: bash {os.path.dirname(os.path.realpath(__file__))}/embed-backfill.sh\n"
-        )
-        sys.exit(1)
-    except (urllib.error.URLError, urllib.error.HTTPError) as e:
-        sys.stderr.write(f"[embed-backfill] FAILED: Ollama unreachable ({e})\n")
-        sys.stderr.write(
-            f"[embed-backfill] re-run via: bash {os.path.dirname(os.path.realpath(__file__))}/embed-backfill.sh\n"
-        )
-        sys.exit(1)
-    embeddings = payload.get("embeddings", [])
-    if not embeddings or not embeddings[0]:
-        sys.stderr.write(f"[embed-backfill] FAILED: Ollama returned no embedding for text length={len(text)}\n")
-        sys.exit(1)
-    return embeddings[0]
-
-
-def cypher_string_literal(s: str) -> str:
-    """Format a Python string as a Cypher single-quoted string literal.
-
-    Escapes the two characters Cypher requires escaping inside single-quoted
-    strings: backslash and single quote. elementId values from Neo4j 5 are
-    typically `<dbprefix>:<uuid>:<recordId>` (alphanumeric + colon + dash) and
-    will not normally contain either, but escape defensively so a future
-    elementId format change cannot break the WRITE batch with a syntax error.
-    """
-    return "'" + s.replace("\\", "\\\\").replace("'", "\\'") + "'"
-
-
-def cypher_float_list(values: list[float]) -> str:
-    """Format a list of floats as a Cypher list literal `[v1, v2, ...]`.
-
-    repr() on a Python float emits a decimal that Cypher accepts as a number
-    literal — including the negative sign, scientific notation, and infinity
-    edge cases. nomic-embed-text returns finite cosine-bounded floats so
-    inf/nan are not expected, but Python's repr is stable for any case that
-    does occur.
-    """
-    return "[" + ",".join(repr(v) for v in values) + "]"
-
-
-# Build the WHERE clause once. The $registered parameter is interpolated
-# into Cypher as a list literal; cypher-shell --param gives us a typed pass.
-COALESCE_TEXT = "coalesce(" + ", ".join(f"n.{p}" for p in EMBED_TEXT_PROPS) + ", '')"
-COUNT_QUERY = f"""
-MATCH (n) WHERE n.embedding IS NULL
-  AND NOT n:Trashed
-  AND any(label IN labels(n) WHERE label IN $registered)
-  AND {COALESCE_TEXT} <> ''
-RETURN count(n) AS remaining;
-"""
-BATCH_QUERY = f"""
-MATCH (n) WHERE n.embedding IS NULL
-  AND NOT n:Trashed
-  AND any(label IN labels(n) WHERE label IN $registered)
-  AND {COALESCE_TEXT} <> ''
-RETURN elementId(n) AS id,
-       labels(n)[0] AS firstLabel,
-       {COALESCE_TEXT} AS text
-LIMIT $batchSize;
-"""
-
-count_out = cypher(COUNT_QUERY, {"registered": REGISTERED_LABELS})
-total_remaining = 0
-for row in parse_csv_rows(count_out):
-    total_remaining = int(row["remaining"])
-
-print(f"[embed-backfill] start total={total_remaining} model={EMBED_MODEL}")
-
-if total_remaining == 0:
-    print("[embed-backfill] done remaining=0 (nothing to backfill)")
-    sys.exit(0)
-
-# Pre-warm Ollama so the first per-node call doesn't pay the model-load
-# latency. The cold-start window for nomic-embed-text on a Pi 5 can exceed
-# 30s; calling once with a tiny throwaway input loads the weights into
-# memory before the loop begins. Failure here is treated identically to
-# any other Ollama failure — loud abort with re-run instruction.
-print(f"[embed-backfill] pre-warm model={EMBED_MODEL} timeout=180s")
-ollama_embed("warmup", timeout=180, retry_on_timeout=False)
-
-processed_total = 0
-batch_index = 0
-while True:
-    batch_start = time.time()
-    batch_out = cypher(
-        BATCH_QUERY,
-        {"registered": REGISTERED_LABELS, "batchSize": BATCH_SIZE},
-    )
-    rows = parse_csv_rows(batch_out)
-    if not rows:
-        break
-
-    # Compute embeddings serially. Ollama on a Pi 5 handles ~3-10 embeds
-    # per second with nomic-embed-text; concurrent requests just queue
-    # behind the GPU/CPU bottleneck so parallelism wouldn't help.
-    pairs: list[tuple[str, list[float]]] = []
-    label_counts: dict[str, int] = {}
-    for row in rows:
-        node_id = row["id"]
-        text = row["text"]
-        first_label = row["firstLabel"]
-        if not text:
-            continue
-        embedding = ollama_embed(text)
-        pairs.append((node_id, embedding))
-        label_counts[first_label] = label_counts.get(first_label, 0) + 1
-
-    if not pairs:
-        # Defensive: query said rows exist but all text was empty after
-        # the python read — means the COALESCE_TEXT predicate is wider
-        # than the python check. Stop to avoid an infinite loop.
-        sys.stderr.write("[embed-backfill] WARN: batch returned rows with empty text — stopping to avoid infinite loop\n")
-        break
-
-    # Build the WRITE batch as a Cypher literal payload rather than a
-    # `--param` map. cypher-shell's `--param` parses the value as a Cypher
-    # expression, and Cypher map keys must be bare identifiers (or backtick-
-    # quoted) — NOT double-quoted strings as JSON would emit. Interpolating
-    # bare-key map literals directly avoids the question entirely:
-    #
-    #   UNWIND [{id: '4:abc:1', embedding: [0.1, 0.2, ...]}, ...] AS pair
-    #   MATCH (n) WHERE elementId(n) = pair.id
-    #   SET n.embedding = pair.embedding;
-    #
-    # cypher_string_literal escapes any backslash/quote in elementIds
-    # defensively; cypher_float_list serialises the embedding via repr()
-    # which Cypher accepts as a number literal.
-    pair_literals = ",".join(
-        f"{{id: {cypher_string_literal(node_id)}, embedding: {cypher_float_list(embedding)}}}"
-        for node_id, embedding in pairs
-    )
-    cypher(
-        f"""
-        UNWIND [{pair_literals}] AS pair
-        MATCH (n) WHERE elementId(n) = pair.id
-        SET n.embedding = pair.embedding;
-        """
-    )
-    elapsed_ms = int((time.time() - batch_start) * 1000)
-    batch_index += 1
-    processed_total += len(pairs)
-    label_summary = ", ".join(f"{k}={v}" for k, v in sorted(label_counts.items()))
-    print(f"[embed-backfill] batch={batch_index} processed={len(pairs)} elapsed-ms={elapsed_ms} labels={label_summary}")
-
-# Final remaining check — should be zero or the diff between original
-# total and processed_total (e.g. if new writes landed mid-run).
-final_out = cypher(COUNT_QUERY, {"registered": REGISTERED_LABELS})
-final_remaining = 0
-for row in parse_csv_rows(final_out):
-    final_remaining = int(row["remaining"])
-print(f"[embed-backfill] done processed={processed_total} remaining={final_remaining}")
-PYEOF