@rubytech/create-realagent 1.0.865 → 1.0.867

package/payload/platform/lib/graph-search/src/index.ts

@@ -92,6 +92,20 @@ export interface ScoredNode {
   properties: Record<string, unknown>;
   vectorScore: number;
   bm25Score: number;
+  /**
+   * Task 967 — true when this node entered the merge map via the BM25 path
+   * with a raw Lucene score > 0 (literal-token match against the universal
+   * fulltext index), or via the keyword-subscriptions property-lookup path.
+   * Set BEFORE bm25 normalisation, so a single-hit set whose normalised
+   * value collapses to 0.0 still carries the flag.
+   *
+   * Read by the vector-threshold filter as the carve-out predicate: rows
+   * with weak cosine survive when `bm25Hit` is true (operator's literal
+   * tokens override the semantic similarity floor). Without this flag the
+   * naive `bm25Score > 0` check would drop every bottom-of-set BM25 row
+   * because min-max normalisation pins the lowest score to 0.
+   */
+  bm25Hit: boolean;
 }
 
 export type SearchMode = "hybrid" | "bm25";
@@ -124,6 +138,23 @@ export interface HybridParams extends Bm25OnlyParams {
    * false so embed failure surfaces loudly.
    */
   degradeOnEmbedFailure?: boolean;
+  /**
+   * Task 967 — minimum raw vector cosine in [0,1] required for a row to
+   * pass the merge step. Rows below the threshold are dropped UNLESS they
+   * also entered via the BM25 path (`bm25Hit === true`) — that carve-out
+   * preserves literal-token matches whose embedding happened to score
+   * mediocre. Counts surface in HybridResponse so callers can render an
+   * "N suppressed — show all" affordance.
+   *
+   * Undefined ⇒ no threshold (legacy behaviour; memory MCP keeps this).
+   * Zero ⇒ explicit "show all" override (admin route's /data and /graph
+   * surfaces use this when the operator clicks "show all").
+   *
+   * Calibration: see `DEFAULT_VECTOR_THRESHOLD` in
+   * `platform/ui/server/routes/admin/graph-search.ts` for the route's
+   * default; the lib itself takes no opinion on the value.
+   */
+  vectorThreshold?: number;
 }
 
 export interface HybridResponse {
@@ -139,6 +170,26 @@ export interface HybridResponse {
    * `expandHops === 0` or no merged results.
    */
   expandMs: number;
+  /**
+   * Task 967 — operator-facing threshold accounting.
+   *
+   * `rawMerged`     count of nodes in the merge map BEFORE the threshold
+   *                 filter ran (vector + BM25 + keyword-subscriptions
+   *                 union, deduped by nodeId).
+   * `suppressed`    count of nodes the threshold filter dropped.
+   *                 Always 0 when `vectorThreshold` is undefined.
+   * `bm25Bypass`    count of nodes that would have been dropped by the
+   *                 vector-cosine floor BUT survived because `bm25Hit`
+   *                 was true. Subset of the rendered set.
+   * `threshold`     the actual threshold applied (echoes the param);
+   *                 null when no filter ran. Drives the "show all"
+   *                 affordance: if null, no banner; if non-null and
+   *                 `suppressed > 0`, banner with count.
+   */
+  rawMerged: number;
+  suppressed: number;
+  bm25Bypass: number;
+  threshold: number | null;
 }
 
 export type EmbedFn = (text: string) => Promise<number[]>;
@@ -332,7 +383,21 @@ export async function hybrid(
       bm25Score: normalised[i] ?? 0,
       related: [],
     }));
-    return { mode: "bm25", results, embedError: msg, expandMs: 0 };
+    // Task 967 — threshold has no meaning in the degraded path: no embed
+    // happened, so there is no vector cosine to compare against. Surface
+    // counts in the response (rawMerged = rendered, suppressed = 0,
+    // bm25Bypass = 0, threshold = null) so the caller's render path is
+    // uniform across hybrid + degraded modes.
+    return {
+      mode: "bm25",
+      results,
+      embedError: msg,
+      expandMs: 0,
+      rawMerged: results.length,
+      suppressed: 0,
+      bm25Bypass: 0,
+      threshold: null,
+    };
   }
 
   const labelToIndex = await discoverIndexes(session);
@@ -350,7 +415,15 @@ export async function hybrid(
       .map((l) => labelToIndex.get(l))
       .filter((idx): idx is string => idx !== undefined);
     if (indexesToQuery.length === 0) {
-      return { mode: "hybrid", results: [], expandMs: 0 };
+      return {
+        mode: "hybrid",
+        results: [],
+        expandMs: 0,
+        rawMerged: 0,
+        suppressed: 0,
+        bm25Bypass: 0,
+        threshold: params.vectorThreshold ?? null,
+      };
     }
   } else {
     indexesToQuery = [...new Set(labelToIndex.values())];
@@ -402,6 +475,7 @@ export async function hybrid(
           properties: plainProperties(node.properties),
           vectorScore: score,
           bm25Score: 0,
+          bm25Hit: false,
         });
       }
     }
@@ -458,6 +532,7 @@ export async function hybrid(
       const existing = scoreMap.get(nodeId);
       if (existing) {
         existing.bm25Score = Math.max(existing.bm25Score, 1.0);
+        existing.bm25Hit = true;
       } else {
         const node = record.get("node") as { properties: Record<string, unknown> };
         scoreMap.set(nodeId, {
@@ -466,17 +541,48 @@ export async function hybrid(
           properties: plainProperties(node.properties),
           vectorScore: 0,
           bm25Score: 1.0,
+          bm25Hit: true,
         });
       }
     }
   }
 
-  // --- Merge & rank ---
-  const merged = [...scoreMap.values()]
-    .map((node) => ({
-      ...node,
-      combinedScore: VECTOR_WEIGHT * node.vectorScore + BM25_WEIGHT * node.bm25Score,
-    }))
+  // --- Merge, threshold, rank ---
+  //
+  // Task 967 — vectorThreshold filters BEFORE sort+slice. Order matters:
+  // slicing first would consume the slot budget with rows the threshold
+  // would have suppressed, under-filling the rendered set. Sorting before
+  // filtering would waste cycles ordering rows we are about to drop. So:
+  // filter -> sort -> slice. Counts (rawMerged / suppressed / bm25Bypass)
+  // are computed against the pre-filter and post-filter populations so the
+  // route can render "N suppressed — show all" without a second query.
+  //
+  // Carve-out: a row with vector cosine below the threshold survives iff
+  // bm25Hit is true. The flag (not bm25Score) is the gate — min-max
+  // normalisation pins the lowest BM25 row's score to 0.0, so checking
+  // bm25Score > 0 would silently drop a literal-token match that just
+  // happens to be the worst-scored BM25 hit in the set.
+  const allMerged = [...scoreMap.values()].map((node) => ({
+    ...node,
+    combinedScore: VECTOR_WEIGHT * node.vectorScore + BM25_WEIGHT * node.bm25Score,
+  }));
+  const rawMerged = allMerged.length;
+  const threshold = params.vectorThreshold;
+  let kept = allMerged;
+  let bm25Bypass = 0;
+  if (threshold !== undefined) {
+    kept = [];
+    for (const node of allMerged) {
+      if (node.vectorScore >= threshold) {
+        kept.push(node);
+      } else if (node.bm25Hit) {
+        kept.push(node);
+        bm25Bypass++;
+      }
+    }
+  }
+  const suppressed = rawMerged - kept.length;
+  const merged = kept
     .sort((a, b) => b.combinedScore - a.combinedScore)
     .slice(0, limit);
 
@@ -561,7 +667,15 @@ export async function hybrid(
     }
   }
 
-  return { mode: "hybrid", results, expandMs };
+  return {
+    mode: "hybrid",
+    results,
+    expandMs,
+    rawMerged,
+    suppressed,
+    bm25Bypass,
+    threshold: threshold ?? null,
+  };
 }
 
 function mergeBm25Hit(
@@ -572,6 +686,11 @@ function mergeBm25Hit(
   const existing = map.get(hit.nodeId);
   if (existing) {
     existing.bm25Score = Math.max(existing.bm25Score, normalisedScore);
+    // Task 967 — set bm25Hit irrespective of normalised value. The hit
+    // came from db.index.fulltext.queryNodes (raw Lucene score > 0 by
+    // construction); min-max normalisation pinning the lowest row to
+    // 0.0 is a presentation artefact, not a "no match" signal.
+    existing.bm25Hit = true;
   } else {
     map.set(hit.nodeId, {
       nodeId: hit.nodeId,
@@ -579,6 +698,7 @@ function mergeBm25Hit(
       properties: hit.properties,
       vectorScore: 0,
       bm25Score: normalisedScore,
+      bm25Hit: true,
     });
   }
 }

package/payload/platform/plugins/admin/skills/publish-site/SKILL.md

@@ -2,6 +2,8 @@
 
 Move an already-extracted static-site tree into the per-account static-publish surface (`<accountDir>/sites/<slug>/`) and emit exactly one canonical path slug for the operator to share. The operator pastes the slug after their own public-host root — the public hostname is whatever their tunnel terminates at, and is not knowable from inside this skill.
 
+**Invoked from `specialists:content-producer`** when the brief carries a host-website / publish-site / put-online intent (Task 966). Admin's IDENTITY.md routes those intents to that specialist on turn 1; running this skill inline on the main admin runner exhausts the 10-turn budget on per-turn ToolSearch + plugin-read discovery before publish-site executes.
+
 ## When to Use
 
 Activate when **both** are true:

package/payload/platform/plugins/admin/skills/unzip-attachment/SKILL.md

@@ -2,6 +2,8 @@
 
 Safely extract a zip archive the admin has uploaded, inventory its contents, and propose one concrete follow-up per entry class — refusing archives that attempt path traversal, symlink escape, or decompression-bomb pathologies.
 
+**Invoked from `specialists:content-producer`** when the archive is a static-site zip and the brief carries a host-website / publish-site / put-online intent (Task 966) — the specialist chains directly into `publish-site` on the extracted tree. For graph-bound archives (LinkedIn export, conversation transcripts, PDFs etc.), invocation routes via `specialists:database-operator` instead.
+
 ## When to Use
 
 Activate when the `[ATTACHMENTS:]` block of the current turn contains an entry whose MIME is `application/zip` or `application/x-zip-compressed`. Every such attachment line carries an `ID: <uuid>  Path: <storagePath>` pair — the `storagePath` is the absolute path to the `.zip` on disk and is the input to this flow.

package/payload/platform/templates/agents/admin/IDENTITY.md

@@ -166,7 +166,8 @@ When the user asks what you can do, answer from the specialist domains, admin-ow
 - Never state a future commitment ("I'll flag", "I'll check", "I'll remind") without immediately creating the mechanism to fulfil it — a scheduled event, a task, or a workflow trigger. A commitment without a backing mechanism is a broken promise.
 - Store everything you learn about the business in the graph — not in files.
 - For document ingestion of any kind — PDFs, text, transcripts, web pages, audio, video, single files, archives — delegate to the `specialists:database-operator` specialist. Include the document path, the document subject (account owner, the business, a third party, etc. — ask if not obvious from context), and the scope (admin/shared/public — ask if not obvious). **Not** `specialists:content-producer`. content-producer produces documents from the populated graph; it does not ingest. The two are opposite movements through the graph and must never be conflated.
-- For ad-hoc graph operations — pruning orphan nodes, deduplicating entities, adding edges, normalising labels, tidying schema drift — delegate to the `specialists:database-operator` specialist. Do not perform these inline; they burn admin-turn token budget and displace the conversational focus. **Not** `specialists:personal-assistant` — PA has no graph-write surface; misdelegation fails at the tool-gate after wasting a turn.
+- For any raw cypher against the memory graph — read or write — delegate to the `specialists:database-operator` specialist. Your read surface is the wrapped tools (`memory-search`, `memory-rank`, `conversation-search`, `profile-read`); your write surface is the schema-aware wrappers (`memory-write`, `memory-update`, `memory-find-candidates`, `memory-delete`). Anything that needs a `MATCH ... RETURN`, a property the wrappers do not expose, or a multi-statement transaction is the operator's surface (e.g. property-name reads against `:CloudflareHostname` / `:Task` / `:Person`, edge-shape audits, dedupe merges, label normalisation, prune passes). Do not perform these inline; admin's `# SCHEMA` block lists labels and edges only — it does not carry the property dictionary, so inline cypher routinely targets non-existent properties and the resulting `01N52` warning is not surfaced. **Not** `specialists:personal-assistant` — PA has no graph surface; misdelegation fails at the tool-gate after wasting a turn.
+- For host-website / publish-site / put-online intents — typically a `.zip` attachment with HTML + assets and a request like "host this website" or "put this online" — delegate immediately to the `specialists:content-producer` specialist on turn 1. Do not `ToolSearch` publish-site, do not `plugin-read` the skill inline, do not pre-scan the zip. The specialist owns the `unzip-attachment` → `publish-site` chain and runs the deterministic Bash flow on its own 10-turn budget; running it inline here exhausts the main-runner budget on per-turn discovery (Task 966 evidence: a 2026-05-10 brochure session hit `error_max_turns total_tool_calls=24` before publish-site executed). **Not** `specialists:database-operator` — static-site zips are extracted to disk for publication, never written to the graph.
 
 ## Proactive Commitment Detection
 

package/payload/platform/templates/specialists/agents/content-producer.md

@@ -1,9 +1,9 @@
 ---
 name: content-producer
-description: "Visual production — reads from the populated graph to produce visual artifacts: image generation, PDF rendering, and component delivery. Delegate when a task requires generating images or saving rendered pages as PDF. **Not** document ingestion — ingestion of any kind routes to `specialists:database-operator`."
-summary: "Produces visual output from your graph — generates images and renders pages to PDF. For example, when you need a cover image for a brief or want to save a rendered page as PDF."
+description: "Visual production and static-site hosting — reads from the populated graph to produce visual artifacts (image generation, PDF rendering, component delivery) and hosts already-prepared static sites by extracting attached archives via unzip-attachment then placing the tree under <accountDir>/sites/<slug>/ via publish-site. Delegate for: generating images, saving rendered pages as PDF, or any 'host this website' / 'publish this site' / 'put this online' intent carrying an HTML+assets archive. **Not** document ingestion — graph ingestion of any kind routes to `specialists:database-operator`. Static-site zips are extracted to disk for publication, never written to the graph."
+summary: "Produces visual output from your graph — generates images, renders pages to PDF, and hosts static websites you upload as a zip. For example, when you need a cover image for a brief, want to save a rendered page as PDF, or upload a brochure zip and ask to put it online."
 model: claude-sonnet-4-6
-tools: mcp__memory__memory-search, mcp__replicate__image-generate, mcp__plugin_playwright_playwright__browser_navigate, mcp__plugin_playwright_playwright__browser_snapshot, mcp__plugin_playwright_playwright__browser_take_screenshot, mcp__plugin_playwright_playwright__browser_pdf_save, mcp__admin__render-component, mcp__admin__file-attach
+tools: Bash, mcp__memory__memory-search, mcp__replicate__image-generate, mcp__plugin_playwright_playwright__browser_navigate, mcp__plugin_playwright_playwright__browser_snapshot, mcp__plugin_playwright_playwright__browser_take_screenshot, mcp__plugin_playwright_playwright__browser_pdf_save, mcp__admin__render-component, mcp__admin__file-attach, mcp__admin__plugin-read
 ---
 
 # Content Producer
@@ -68,6 +68,20 @@ Generate PDFs from rendered HTML pages using the browser tools.
 
 **Local files:** `file://` URLs are rewritten transparently by the `playwright-file-guard` PreToolUse hook — pass them directly to `browser_navigate` and the hook will spawn a loopback `python3 -m http.server` on a free port and rewrite the URL before Playwright sees it. No agent-side server management needed.
 
+## Hosting websites
+
+When a brief carries a "host this website" / "publish this site" / "put this online" intent — typically with a `.zip` attachment whose extracted contents are HTML + assets — execute the deterministic two-skill chain. This is **not** ingestion (no graph write); it is filesystem extraction followed by a placement move into the per-account static-publish surface served by the existing `/sites/*` route. The `## Out of scope: ingestion of any kind` rule above is preserved by the SKILL gate at [publish-site SKILL.md](../../../plugins/admin/skills/publish-site/SKILL.md): only HTML + assets directory trees activate this path; PDFs, slide decks, single-image attachments, or zips whose extracted output is not a static-site tree route to their own skills (`a4-print-documents`, `deck-pages`) or — for graph-bound content — back to `specialists:database-operator`.
+
+The chain (≤4 turns, total):
+
+1. **Read the skill texts** via `mcp__admin__plugin-read` — load `admin/skills/unzip-attachment/SKILL.md` and `admin/skills/publish-site/SKILL.md` into context. Both ship invariants (zip-slip guard, declared-uncompressed cap, slug regex, refusal taxonomy) that are mechanically enforced by their shell primitives — read, do not paraphrase.
+2. **Extract** the attached archive via the `unzip-attachment` skill's deterministic Bash flow. Output lands at `<accountDir>/extracted/<attachmentId>/`. Refusals (oversize, zip-slip, symlink, unreadable) are loud-fail; relay the operator message verbatim and stop.
+3. **Confirm the slug** with the operator if not already explicit. The slug is one or more `/`-separated segments under `<accountDir>/sites/`; each segment matches `/^[a-z0-9_][a-z0-9_.-]{0,99}$/i` and no segment starts with `.` or equals `..`. Never invent a slug.
+4. **Publish** via the `publish-site` skill's deterministic Bash flow — single `mv` from the extracted tree into `<accountDir>/sites/<slug>/`. Refusals (`unsafe-slug`, `destination-occupied`, `symlink-in-source`, `zero-html`, `ambiguous-html`) are loud-fail; relay the operator message verbatim and stop.
+5. **Emit** the canonical path slug (`/sites/<slug>/` when an `index.html` is present, otherwise `/sites/<slug>/<file>.html`). One line, framed as "paste this after your public-host root" — no scheme, no host. The `/sites/*` route at [server/routes/sites.ts](../../../ui/server/routes/sites.ts) takes care of the directory-form trailing-slash redirect.
+
+**No fallback servers, no Playwright probes, no service restarts.** If the move or URL form is wrong, refuse — never reach for `python -m http.server`, `npx http-server`, browser-automation probes, or platform restarts. That is the exact failure pattern publish-site exists to close.
+
 ## File delivery
 
 Use `file-attach` to make generated files available for download in the chat. Use `render-component` with component name `file-attachment` for download delivery.

package/payload/platform/templates/specialists/agents/database-operator.md

@@ -1,6 +1,6 @@
 ---
 name: database-operator
-description: "Document and archive ingestion and ad-hoc graph operations — running the universal `document-ingest` skill for any unstructured document (PDF, text, transcript, web page, audio, video) and per-source archive-import skills (LinkedIn Basic Data Export today; CRM-type seed archives as each plugin ships), plus operator-driven graph hygiene (prune orphans, deduplicate entities, add edges, normalise labels). Delegate when the operator uploads any document, drops an archive directory into chat, or asks for any graph operation that is not a routine per-turn write."
+description: "Owner of the memory graph — any raw cypher (read or write) against Neo4j routes here, plus all document and archive ingestion (running the universal `document-ingest` skill for unstructured documents — PDF, text, transcript, web page, audio, video — and per-source archive-import skills — LinkedIn Basic Data Export today; CRM-type seed archives as each plugin ships), plus operator-driven graph hygiene (prune orphans, deduplicate entities, add edges, normalise labels), plus one-off raw reads (property-name lookups, edge-shape audits, multi-statement queries) when admin's wrapped read tools — `memory-search`, `memory-rank`, `conversation-search`, `profile-read` — do not expose the property or relationship being asked about. Delegate when the operator uploads any document, drops an archive directory into chat, asks for any graph operation that is not a routine per-turn wrapped write, or asks a factual question whose answer requires a property or relationship admin's wrapped read tools cannot reach."
 summary: "Ingests every unstructured document and external archive into your graph (LinkedIn today; other CRM sources in future) and handles ad-hoc graph tidy-ups on request. For example, when you upload a CV, a pricing guide, or a contract; when you drop a LinkedIn export folder into chat; or when you ask to prune orphan nodes, merge duplicate people, or add edges between entities."
 model: claude-sonnet-4-6
 tools: Read, Bash, Glob, Grep, mcp__graph__maxy-graph-read_neo4j_cypher, mcp__graph__maxy-graph-write_neo4j_cypher, mcp__graph__maxy-graph-get_neo4j_schema, mcp__memory__memory-write, mcp__memory__memory-update, mcp__memory__memory-delete, mcp__memory__memory-search, mcp__memory__memory-rank, mcp__memory__memory-reindex, mcp__memory__memory-find-candidates, mcp__memory__memory-ingest, mcp__memory__memory-ingest-extract, mcp__memory__memory-ingest-web, mcp__memory__memory-classify, mcp__memory__memory-archive-write, mcp__memory__graph-prune-denylist-list, mcp__memory__graph-prune-denylist-add, mcp__memory__graph-prune-denylist-remove, mcp__contacts__contact-create, mcp__contacts__contact-update, mcp__contacts__contact-lookup, mcp__contacts__contact-list, mcp__tasks__task-create, mcp__admin__file-attach, mcp__admin__plugin-read