clementine-agent 1.18.12 → 1.18.14

package/dist/agent/tool-router.js

@@ -126,6 +126,15 @@ export const TOOL_BUNDLES = [
 function uniqueStrings(values) {
     return [...new Set([...values].filter((v) => !!v && v.trim().length > 0))];
 }
+function explicitMcpServers(scopeText) {
+    const servers = new Set();
+    const re = /\bmcp__([A-Za-z0-9_-]+)__[A-Za-z0-9_.:-]+\b/g;
+    let match;
+    while ((match = re.exec(scopeText)) !== null) {
+        servers.add(match[1]);
+    }
+    return uniqueStrings(servers);
+}
 export function routeToolSurface(text) {
     const scopeText = text?.trim() ?? '';
     if (!scopeText) {
@@ -162,13 +171,26 @@ export function routeToolSurface(text) {
             composio.add(slug);
         inheritFullClaudeEnv = inheritFullClaudeEnv || bundle.inheritFullClaudeEnv === true;
     }
+    for (const server of explicitMcpServers(scopeText)) {
+        if (server.startsWith('claude_ai_')) {
+            external.add(server.slice('claude_ai_'.length));
+        }
+        else {
+            // Exact `mcp__<server>__<tool>` mentions are authoritative. Add the
+            // name as both a direct MCP server and a Composio toolkit; whichever
+            // source is actually connected will mount, and the other path no-ops.
+            external.add(server);
+            composio.add(server);
+        }
+        inheritFullClaudeEnv = true;
+    }
     return {
         bundles: uniqueStrings(bundles),
         externalMcpServers: uniqueStrings(external),
         composioToolkits: uniqueStrings(composio),
         inheritFullClaudeEnv,
         fullSurface: false,
-        reason: bundles.size > 0 ? 'matched' : 'empty',
+        reason: bundles.size > 0 || external.size > 0 || composio.size > 0 ? 'matched' : 'empty',
     };
 }
 //# sourceMappingURL=tool-router.js.map

package/dist/brain/adapters/markdown.js

@@ -33,8 +33,13 @@ export async function* parseMarkdown(filePath) {
         mtime = statSync(filePath).mtime.toISOString();
     }
     catch { /* ignore */ }
+    const frontmatterExternalId = typeof parsed.data?.externalId === 'string' && parsed.data.externalId.trim()
+        ? parsed.data.externalId.trim()
+        : typeof parsed.data?.external_id === 'string' && parsed.data.external_id.trim()
+            ? parsed.data.external_id.trim()
+            : null;
     yield {
-        externalId: `md-${hint}-${contentHash(body)}`,
+        externalId: frontmatterExternalId ?? `md-${hint}-${contentHash(body)}`,
         content: body,
         rawPayload: raw,
         metadata: {

package/dist/brain/connector-recipes.d.ts

@@ -3,8 +3,8 @@
  *
  * Each recipe is a blueprint for a one-click "auto-seed feed" that turns an
  * authenticated tool source (Claude Desktop connector, Composio toolkit, or
- * local MCP server) into a scheduled data feed that writes into the brain's
- * ingest folder.
+ * local MCP server) into a scheduled data feed that writes distilled notes
+ * into the brain's ingest folder.
  *
  * A feed materializes as:
  *   1. A CRON.md job entry with `managed: connector-feed` frontmatter
@@ -12,8 +12,8 @@
  *
  * The cron prompt tells the Claude Code agent to use the integration's MCP
  * tools to pull records, compare them with current memory when appropriate,
- * then call `brain_ingest_folder` to commit them — which writes markdown files
- * and runs the distillation pipeline in one step.
+ * then call `brain_ingest_folder` to commit them — which writes distilled
+ * markdown notes and indexes them in one step.
  *
  * Field syntax in prompt templates:
  *   {{fieldKey}}   — user-supplied value
@@ -59,7 +59,7 @@ export interface ConnectorRecipe {
     description: string;
     /** Emoji shown next to the label. */
     icon: string;
-    /** Matches the key in ~/.clementine/claude-integrations.json */
+    /** Matches the tool source name; "*" recipes are offered for every source. */
     integration: string;
     /** Tools we rely on for this recipe. Used only to warn if the integration
      *  hasn't surfaced them yet in claude-integrations.json. */

package/dist/brain/connector-recipes.js

@@ -3,8 +3,8 @@
  *
  * Each recipe is a blueprint for a one-click "auto-seed feed" that turns an
  * authenticated tool source (Claude Desktop connector, Composio toolkit, or
- * local MCP server) into a scheduled data feed that writes into the brain's
- * ingest folder.
+ * local MCP server) into a scheduled data feed that writes distilled notes
+ * into the brain's ingest folder.
  *
  * A feed materializes as:
  *   1. A CRON.md job entry with `managed: connector-feed` frontmatter
@@ -12,8 +12,8 @@
  *
  * The cron prompt tells the Claude Code agent to use the integration's MCP
  * tools to pull records, compare them with current memory when appropriate,
- * then call `brain_ingest_folder` to commit them — which writes markdown files
- * and runs the distillation pipeline in one step.
+ * then call `brain_ingest_folder` to commit them — which writes distilled
+ * markdown notes and indexes them in one step.
  *
  * Field syntax in prompt templates:
  *   {{fieldKey}}   — user-supplied value
@@ -25,16 +25,116 @@ function slugify(s) {
         .replace(/^-+|-+$/g, '')
         .slice(0, 40) || 'feed';
 }
+function inferToolServer(toolName) {
+    const match = String(toolName).match(/^mcp__([^_]+(?:_[^_]+)*)__/);
+    return match?.[1] ?? 'tool';
+}
 const COMMIT_INSTRUCTIONS = `When you have the records collected, call the \`brain_ingest_folder\` MCP tool with:
 - \`slug\`: "{{slug}}"
 - \`records\`: an array of \`{title, externalId, content, metadata}\` objects (one per item). \`externalId\` should be the source provider's stable id so re-runs dedup. \`metadata\` can include any fields you want preserved (url, modifiedAt, author).
 
-That tool writes each record to \`{{targetFolder}}/\` and runs the brain's distillation pipeline. You do NOT need to use Write — brain_ingest_folder handles file creation. Finish by reporting a one-line summary like "Ingested N new records, M unchanged".
+That tool runs the brain's distillation pipeline and writes the final notes to \`{{targetFolder}}/\`. You do NOT need to use Write — brain_ingest_folder handles note creation and indexing. Finish by reporting a one-line summary like "Ingested N new records, M unchanged".
 
 If the tool returns an error, include the error text in your summary.`;
-const MEMORY_DELTA_INSTRUCTIONS = `Before committing, call \`memory_recall\` for the feed slug/topic and use the returned chunks as the current memory state for this source. Keep records that are new, materially changed, or contain a new finding. Drop exact duplicates and rows that add no useful information. The ingestion pipeline will write markdown and embeddings; do not call \`memory_write\` for these feed records.`;
+const MEMORY_DELTA_INSTRUCTIONS = `Before committing, call \`memory_recall\` for the feed slug/topic and use the returned chunks as the current memory state for this source. Keep records that are new, materially changed, or contain a new finding. Drop exact duplicates and rows that add no useful information. The ingestion pipeline will write markdown, chunk it, and index it for recall; do not call \`memory_write\` for these feed records.`;
 // ── Recipes ────────────────────────────────────────────────────────────
 export const RECIPES = [
+    {
+        id: 'tool-backed-memory-seed',
+        label: 'Seed memory from this tool',
+        description: 'Pick one tool, fetch records from it, compare them with current memory, and save only new or changed findings.',
+        icon: '🔌',
+        integration: '*',
+        requiredTools: [],
+        fields: [
+            {
+                key: 'topic',
+                label: 'Memory topic',
+                placeholder: 'customers, calls, leads, deals, meetings...',
+                required: true,
+                help: 'Used to search current memory and name this feed.',
+            },
+            {
+                key: 'toolName',
+                label: 'Tool to call',
+                required: true,
+                help: 'Pick the exact tool this feed should call each time it runs.',
+            },
+            {
+                key: 'callGoal',
+                label: 'What should Clementine fetch?',
+                placeholder: 'Fetch updated HubSpot contacts modified since the last run...',
+                required: true,
+                help: 'Describe the records to fetch, filters to apply, and any pagination bounds.',
+            },
+            {
+                key: 'variablesJson',
+                label: 'Tool variables (JSON)',
+                placeholder: '{"listId":"123","limit":100,"updatedAfter":"last_run"}',
+                help: 'Optional. Use {} if the tool needs no arguments.',
+            },
+            {
+                key: 'recordStrategy',
+                label: 'How to save each result',
+                placeholder: 'One record per contact. Use email as stable id. Summarize lifecycle stage, owner, last activity, and new changes.',
+                help: 'Tell Clementine what counts as one memory record and which field is the stable id.',
+            },
+            {
+                key: 'slug',
+                label: 'Memory bucket name (optional)',
+                placeholder: 'hubspot-contacts',
+                help: 'Optional. Leave blank to derive one from the connector and topic.',
+            },
+            {
+                key: 'limit',
+                label: 'Max records per run',
+                placeholder: '100',
+                defaultValue: '100',
+            },
+        ],
+        defaultSchedule: '0 8 * * *',
+        tier: 2,
+        slugFromValues: (v) => `tool-${slugify(v.slug || `${v.toolSourceName || inferToolServer(v.toolName || '')}-${v.topic || v.toolName || 'feed'}`)}`,
+        buildPrompt: (v, ctx) => {
+            const sourceName = v.toolSourceName || inferToolServer(v.toolName || '');
+            const sourceKind = v.toolSourceKind || 'mcp';
+            const sourceLabel = v.toolSourceLabel || sourceName;
+            const topic = v.topic || 'tool-backed memory';
+            const limit = v.limit || '100';
+            return `You are running a generic tool-backed memory seed feed.
+
+Tool source:
+- Label: "${sourceLabel}"
+- Source name: "${sourceName}"
+- Source kind: "${sourceKind}"
+- Tool: \`${v.toolName}\`
+
+Goal: ${v.callGoal || `Call ${v.toolName} and ingest useful returned data into memory.`}
+
+Tool variables JSON:
+\`\`\`json
+${(v.variablesJson || '{}').trim() || '{}'}
+\`\`\`
+
+How to save each result:
+${v.recordStrategy || 'Convert the tool response into one memory record per returned entity or event. Use the provider stable id when available; otherwise use a deterministic hash of the source, topic, and meaningful record key.'}
+
+Steps:
+1. Call exactly this selected tool: \`${v.toolName}\`. Use the Tool variables JSON and the Goal above as the tool-call inputs. If the tool schema needs differently named arguments, map the provided variables to that schema. Do not switch to a different external tool unless this tool returns a clear instruction that another tool is required to read the selected records.
+2. If the tool supports pagination or modified-since filters, prefer new/updated records and stop after ${limit} records. If no modified-since filter is available, fetch the most relevant ${limit} records.
+3. Normalize the tool result into candidate records. Preserve stable ids, URLs, timestamps, owners/authors, status fields, and provider metadata. Skip empty or purely administrative records.
+4. ${MEMORY_DELTA_INSTRUCTIONS}
+   Use this recall query: \`source:${ctx.slug} ${topic} ${sourceLabel} ${v.toolName}\`.
+5. Compare the normalized candidates with recalled memory. Keep only candidates that are new, materially changed, or produce a new useful finding. Drop exact duplicates and trivial timestamp-only changes unless the timestamp itself is the useful fact.
+6. For each kept candidate, build one record:
+   - \`title\`: a compact human label including the topic and record name/id.
+   - \`externalId\`: \`${sourceName}:${topic}:<providerStableIdOrDeterministicHash>\`.
+   - \`content\`: markdown containing the current facts, the new/changed finding, and a "Source data" section with relevant returned fields.
+   - \`metadata\`: \`{provider:"${sourceName}", toolSource:"${sourceKind}", toolName:"${v.toolName}", topic:"${topic}", fetchedAt, sourceUrl, updatedAt}\` plus any provider-specific keys worth preserving.
+7. ${COMMIT_INSTRUCTIONS.replace(/{{slug}}/g, ctx.slug).replace(/{{targetFolder}}/g, ctx.targetFolder)}
+`;
+        },
+    },
     {
         id: 'gdrive-watch-folder',
         label: 'Google Drive: watch a folder',

package/dist/cli/dashboard.js

@@ -3834,7 +3834,8 @@ export async function cmdDashboard(opts) {
     // the Intelligence → Sources tab composes recipe + field values + schedule
     // into a cron prompt that uses the user's authenticated tool source
     // (Claude Desktop connector, Composio toolkit, or local MCP server) to pull
-    // records and calls brain_ingest_folder to commit them.
+    // records, compare them with memory, and call brain_ingest_folder to commit
+    // distilled notes.
     app.get('/api/brain/connectors', async (_req, res) => {
         try {
             const { getClaudeIntegrations, loadToolInventory } = await import('../agent/mcp-bridge.js');
@@ -3890,7 +3891,7 @@ export async function cmdDashboard(opts) {
                     const connected = await composio.listConnectedToolkits();
                     const activeSlugs = [...new Set(connected
                             .filter((c) => c.status === 'ACTIVE')
-                            .filter((c) => recipeIntegrations.has(c.slug))
+                            .filter((c) => recipeIntegrations.has('*') || recipeIntegrations.has(c.slug))
                             .map((c) => c.slug))];
                     if (activeSlugs.length) {
                         const { listComposioToolkitTools } = await import('../integrations/composio/mcp-bridge.js');
@@ -4083,6 +4084,27 @@ If the tool returns nothing or errors, return an empty array \`[]\`.`,
                 res.status(400).json({ error: `missing required field(s): ${missing.join(', ')}` });
                 return;
             }
+            if (recipe.id === 'tool-backed-memory-seed') {
+                const toolName = String(values.toolName ?? '').trim();
+                if (!/^mcp__.+__.+$/.test(toolName)) {
+                    res.status(400).json({ error: 'toolName must be an exact MCP tool name like mcp__server__tool' });
+                    return;
+                }
+                const rawVariables = String(values.variablesJson ?? '').trim();
+                if (rawVariables) {
+                    try {
+                        const parsedVariables = JSON.parse(rawVariables);
+                        if (!parsedVariables || typeof parsedVariables !== 'object' || Array.isArray(parsedVariables)) {
+                            res.status(400).json({ error: 'Tool variables must be a JSON object, for example {}' });
+                            return;
+                        }
+                    }
+                    catch {
+                        res.status(400).json({ error: 'Tool variables must be valid JSON, for example {}' });
+                        return;
+                    }
+                }
+            }
             const schedule = (body.schedule || recipe.defaultSchedule).trim();
             if (!cron.validate(schedule)) {
                 res.status(400).json({ error: `invalid cron expression: ${schedule}` });
@@ -4119,7 +4141,7 @@ If the tool returns nothing or errors, return an empty array \`[]\`.`,
                         managed: 'connector-feed',
                         recipeId: recipe.id,
                         fields: values,
-                        inputPath: path.join(VAULT_DIR, spec.targetFolder),
+                        mode: 'direct-records',
                     }),
                     targetFolder: spec.targetFolder,
                     intelligence: 'auto',
@@ -13549,11 +13571,11 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
           <!-- ═══ Auto-seed feeds (connected tools → cron → brain) ═══ -->
           <div class="card" style="padding:16px;margin-bottom:16px">
             <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:4px">
-              <div style="font-weight:600">Auto-seed feeds</div>
+              <div style="font-weight:600">Seed memory from connected apps</div>
               <button class="btn-primary" onclick="brainOpenFeedWizard()">+ Add feed</button>
             </div>
             <div style="color:var(--muted);font-size:13px;margin-bottom:12px">
-              One-click scheduled feeds that use authenticated tools (Composio, Claude Desktop connectors, or local MCP servers) to pull records and commit them to the brain.
+              Scheduled feeds use authenticated tools (Composio, Claude Desktop connectors, or local MCP servers) to fetch records, compare them with current memory, and save distilled notes to the brain.
             </div>
             <div id="brain-feeds-connectors" style="display:flex;flex-wrap:wrap;gap:6px;margin-bottom:12px"></div>
             <div id="brain-feeds-list"></div>
@@ -13561,7 +13583,7 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
 
           <!-- ═══ Auto-seed feed wizard (hidden by default) ═══ -->
           <div id="brain-feed-wizard" class="card" style="display:none;padding:16px;margin-bottom:16px">
-            <div style="font-weight:600;margin-bottom:4px">Add auto-seed feed</div>
+            <div style="font-weight:600;margin-bottom:4px">Add memory seed feed</div>
             <div id="brain-feed-wizard-breadcrumbs" style="color:var(--muted);font-size:12px;margin-bottom:12px"></div>
             <div id="brain-feed-wizard-step"></div>
             <div style="display:flex;gap:8px;margin-top:14px">
@@ -14115,6 +14137,11 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
             for (const f of (s.recipe.fields || [])) {
               if (f.defaultValue) s.values[f.key] = f.defaultValue;
             }
+            if (s.recipe.integration === '*' && s.pick) {
+              s.values.toolSourceName = s.pick.name;
+              s.values.toolSourceKind = s.pick.kind;
+              s.values.toolSourceLabel = s.pick.label;
+            }
             s.schedule = s.recipe.defaultSchedule;
             s.step = 2;
           } else if (s.step === 2) {
@@ -14123,6 +14150,27 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
             inputs.forEach(function(inp) { s.values[inp.dataset.field] = inp.value; });
             const missing = (s.recipe.fields || []).filter(function(f) { return f.required && !(s.values[f.key] || '').trim(); });
             if (missing.length) { document.getElementById('brain-feed-wizard-status').innerHTML = '<span style="color:#e66">Required: ' + missing.map(function(f) { return f.label; }).join(', ') + '</span>'; return; }
+            if (s.recipe && s.recipe.id === 'tool-backed-memory-seed') {
+              const toolName = String(s.values.toolName || '').trim();
+              if (!/^mcp__.+__.+$/.test(toolName)) {
+                document.getElementById('brain-feed-wizard-status').innerHTML = '<span style="color:#e66">Pick an exact tool before continuing.</span>';
+                return;
+              }
+              const rawVariables = String(s.values.variablesJson || '').trim();
+              if (rawVariables) {
+                try {
+                  const parsedVariables = JSON.parse(rawVariables);
+                  if (!parsedVariables || typeof parsedVariables !== 'object' || Array.isArray(parsedVariables)) {
+                    document.getElementById('brain-feed-wizard-status').innerHTML = '<span style="color:#e66">Tool variables must be a JSON object, for example {}.</span>';
+                    return;
+                  }
+                } catch (err) {
+                  void err;
+                  document.getElementById('brain-feed-wizard-status').innerHTML = '<span style="color:#e66">Tool variables must be valid JSON, for example {}.</span>';
+                  return;
+                }
+              }
+            }
             s.step = 3;
           } else if (s.step === 3) {
             brainFeedWizardSubmit();
@@ -14290,6 +14338,13 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
           if (field) await brainRenderFieldPicker(field, s.values);
         }
 
+        function brainFullToolNameForPick(pick, tool) {
+          if (!pick || !tool) return tool || '';
+          if (String(tool).startsWith('mcp__')) return tool;
+          const server = pick.kind === 'claude-desktop' ? ('claude_ai_' + pick.name) : pick.name;
+          return 'mcp__' + server + '__' + tool;
+        }
+
         function brainFeedWizardRender() {
           if (!brainFeedWizardState) return;
           const s = brainFeedWizardState;
@@ -14317,7 +14372,7 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
                 }).join('') + '</div>';
             }
           } else if (s.step === 1) {
-            const recipes = (s.catalog.recipes || []).filter(function(r) { return r.integration === s.pick.name; });
+            const recipes = (s.catalog.recipes || []).filter(function(r) { return r.integration === s.pick.name || r.integration === '*'; });
             if (!recipes.length) {
               html = '<div style="color:var(--muted)">No recipes for this connector yet.</div>';
             } else {
@@ -14346,6 +14401,25 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
                       '<div style="color:var(--muted);font-size:13px;padding:6px">Loading choices…</div>' +
                     '</div>' +
                     '<input type="hidden" data-field="' + f.key + '" value="' + escapeHtml(val) + '">';
+                  } else if (s.recipe.integration === '*' && f.key === 'toolName') {
+                    const tools = (s.pick && s.pick.tools) || [];
+                    if (!tools.length) {
+                      control = '<input type="text" data-field="' + f.key + '" value="' + escapeHtml(val) + '" placeholder="mcp__server__TOOL_NAME" style="width:100%">';
+                    } else {
+                      const options = tools.map(function(t) {
+                        const full = brainFullToolNameForPick(s.pick, t);
+                        const selected = full === val ? ' selected' : '';
+                        return '<option value="' + escapeHtml(full) + '"' + selected + '>' + escapeHtml(t) + '</option>';
+                      }).join('');
+                      control = '<select data-field="' + f.key + '" style="width:100%;padding:6px">' +
+                        '<option value="">— pick a tool —</option>' +
+                        options +
+                      '</select>' +
+                      '<div style="font-size:11px;color:var(--muted);margin-top:4px">The feed will call the selected tool exactly, then compare returned records with memory.</div>';
+                    }
+                  } else if (s.recipe.integration === '*' && ['callGoal', 'variablesJson', 'recordStrategy'].includes(f.key)) {
+                    const minHeight = f.key === 'variablesJson' ? '70px' : '92px';
+                    control = '<textarea data-field="' + f.key + '" placeholder="' + escapeHtml(f.placeholder || '') + '" style="width:100%;min-height:' + minHeight + ';resize:vertical">' + escapeHtml(val) + '</textarea>';
                   } else {
                     control = '<input type="text" data-field="' + f.key + '" value="' + escapeHtml(val) + '" placeholder="' + escapeHtml(f.placeholder || '') + '" style="width:100%">';
                   }
@@ -14387,6 +14461,11 @@ if('serviceWorker' in navigator){navigator.serviceWorker.getRegistrations().then
         function brainFeedWizardPickRecipe(id) {
           const r = (brainFeedWizardState.catalog.recipes || []).find(function(x) { return x.id === id; });
           brainFeedWizardState.recipe = r;
+          if (r && r.integration === '*' && brainFeedWizardState.pick) {
+            brainFeedWizardState.values.toolSourceName = brainFeedWizardState.pick.name;
+            brainFeedWizardState.values.toolSourceKind = brainFeedWizardState.pick.kind;
+            brainFeedWizardState.values.toolSourceLabel = brainFeedWizardState.pick.label;
+          }
           brainFeedWizardRender();
         }
 

package/dist/tools/brain-tools.d.ts

@@ -3,9 +3,33 @@
  *
  * Tools the agent uses to feed the brain's ingestion pipeline from cron jobs.
  * Primarily used by Connector Feeds (src/brain/connector-recipes.ts) — each
- * feed's cron prompt ends with a brain_ingest_folder call that writes fetched
- * records to 04-Ingest/<slug>/ and runs distillation.
+ * feed's cron prompt ends with a brain_ingest_folder call that sends fetched
+ * records into the distillation pipeline. The pipeline writes distilled notes
+ * to 04-Ingest/<slug>/ and indexes them for recall.
  */
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+export interface IngestRecordInput {
+    title: string;
+    externalId: string;
+    content: string;
+    metadata?: Record<string, unknown>;
+}
+export interface BrainIngestFolderResult {
+    slug: string;
+    acceptedCount: number;
+    skippedEmpty: number;
+    pipeline: {
+        recordsIn: number;
+        recordsWritten: number;
+        recordsSkipped: number;
+        recordsFailed: number;
+        errors: Array<{
+            externalId?: string;
+            error: string;
+        }>;
+    };
+    message: string;
+}
+export declare function ingestBrainRecords(slug: string, records: IngestRecordInput[]): Promise<BrainIngestFolderResult>;
 export declare function registerBrainTools(server: McpServer): void;
 //# sourceMappingURL=brain-tools.d.ts.map

package/dist/tools/brain-tools.js

@@ -3,31 +3,19 @@
  *
  * Tools the agent uses to feed the brain's ingestion pipeline from cron jobs.
  * Primarily used by Connector Feeds (src/brain/connector-recipes.ts) — each
- * feed's cron prompt ends with a brain_ingest_folder call that writes fetched
- * records to 04-Ingest/<slug>/ and runs distillation.
+ * feed's cron prompt ends with a brain_ingest_folder call that sends fetched
+ * records into the distillation pipeline. The pipeline writes distilled notes
+ * to 04-Ingest/<slug>/ and indexes them for recall.
  */
-import { mkdirSync, writeFileSync, existsSync } from 'node:fs';
-import path from 'node:path';
 import { z } from 'zod';
-import { VAULT_DIR, logger, textResult } from './shared.js';
-/** Slugify a record title for the filename — URL-safe, short, collision-resistant with externalId. */
-function filenameFor(title, externalId) {
-    const base = String(title || externalId || 'record')
-        .toLowerCase()
-        .replace(/[^a-z0-9]+/g, '-')
-        .replace(/^-+|-+$/g, '')
-        .slice(0, 60) || 'record';
-    const idPart = String(externalId || '')
-        .replace(/[^a-zA-Z0-9]+/g, '')
-        .slice(0, 16) || 'x';
-    return `${base}-${idPart}.md`;
-}
-function formatFrontmatter(record, slug) {
+import { fallbackExternalId } from '../brain/adapters/common.js';
+import { logger, textResult } from './shared.js';
+function formatFrontmatter(record, slug, fetchedAt) {
     const frontmatter = {
         source: slug,
         externalId: record.externalId,
         title: record.title,
-        fetchedAt: new Date().toISOString(),
+        fetchedAt,
     };
     if (record.metadata && typeof record.metadata === 'object') {
         for (const [k, v] of Object.entries(record.metadata)) {
@@ -51,8 +39,94 @@ function formatFrontmatter(record, slug) {
     lines.push('---');
     return lines.join('\n') + '\n\n';
 }
+function sanitizeSlug(slug) {
+    return String(slug).toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/^-+|-+$/g, '');
+}
+function toRawRecords(records, slug) {
+    const fetchedAt = new Date().toISOString();
+    const rawRecords = [];
+    let skippedEmpty = 0;
+    for (const [index, record] of records.entries()) {
+        const content = String(record.content ?? '').trim();
+        if (!content) {
+            skippedEmpty += 1;
+            continue;
+        }
+        const title = String(record.title || record.externalId || `Record ${index + 1}`).trim();
+        const externalId = String(record.externalId || '').trim()
+            || fallbackExternalId(`${slug}-record`, index + 1, content);
+        const normalized = {
+            title,
+            externalId,
+            content,
+            metadata: record.metadata,
+        };
+        rawRecords.push({
+            externalId,
+            content,
+            rawPayload: formatFrontmatter(normalized, slug, fetchedAt) + content,
+            metadata: {
+                ...(record.metadata ?? {}),
+                adapter: 'connector-feed',
+                source: slug,
+                externalId,
+                title,
+                fetchedAt,
+            },
+        });
+    }
+    return { rawRecords, skippedEmpty };
+}
+async function* iterateRecords(records) {
+    for (const record of records)
+        yield record;
+}
+export async function ingestBrainRecords(slug, records) {
+    const safeSlug = sanitizeSlug(slug);
+    if (!safeSlug)
+        throw new Error('slug is required');
+    if (!Array.isArray(records) || records.length === 0)
+        throw new Error(`no records to ingest for slug "${safeSlug}"`);
+    const { rawRecords, skippedEmpty } = toRawRecords(records, safeSlug);
+    if (rawRecords.length === 0)
+        throw new Error(`no non-empty records to ingest for slug "${safeSlug}"`);
+    const { upsertSource, getSource } = await import('../brain/source-registry.js');
+    const { runIngestion } = await import('../brain/ingestion-pipeline.js');
+    await upsertSource({
+        slug: safeSlug,
+        kind: 'seed',
+        adapter: 'markdown',
+        configJson: JSON.stringify({ managed: 'connector-feed', mode: 'direct-records' }),
+        targetFolder: `04-Ingest/${safeSlug}`,
+        intelligence: 'auto',
+        enabled: true,
+    });
+    const source = await getSource(safeSlug);
+    if (!source)
+        throw new Error('failed to register source');
+    const result = await runIngestion({ source, records: iterateRecords(rawRecords) });
+    let ingestionSummary = `Pipeline: ${result.recordsIn} in · ${result.recordsWritten} written · ${result.recordsSkipped} skipped · ${result.recordsFailed} failed`;
+    if (result.errors?.length) {
+        ingestionSummary += ` (first error: ${result.errors[0].error.slice(0, 100)})`;
+    }
+    const message = `Ingested into slug "${safeSlug}": ${rawRecords.length} accepted record(s), ${skippedEmpty} empty skipped. ${ingestionSummary}`;
+    logger.info({ slug: safeSlug, acceptedCount: rawRecords.length, skippedEmpty, recordCount: records.length }, 'brain_ingest_folder complete');
+    return {
+        slug: safeSlug,
+        acceptedCount: rawRecords.length,
+        skippedEmpty,
+        pipeline: {
+            recordsIn: result.recordsIn,
+            recordsWritten: result.recordsWritten,
+            recordsSkipped: result.recordsSkipped,
+            recordsFailed: result.recordsFailed,
+            errors: result.errors,
+        },
+        message,
+    };
+}
 export function registerBrainTools(server) {
-    server.tool('brain_ingest_folder', 'Ingest a batch of records into the brain under a named slug. Writes each record as a markdown file in 04-Ingest/<slug>/ with frontmatter, then runs the distillation pipeline (chunking, LLM summarization, vault note write, knowledge graph write). Use at the end of Connector Feed cron jobs. Safe to re-run — existing files with matching content hashes are deduped by the pipeline.', {
+    server.tool('brain_ingest_folder', 'Ingest a batch of records into the brain under a named slug. Sends records directly into the distillation pipeline (chunking, LLM summarization, vault note write, memory indexing, knowledge graph write). Use at the end of Connector Feed cron jobs. Safe to re-run — records with the same externalId update the same distilled note.', {
         slug: z.string().describe('Feed slug (matches 04-Ingest/<slug> folder). Lowercase, hyphen-separated.'),
         records: z.array(z.object({
             title: z.string().describe('Human-readable title for this record.'),
@@ -67,64 +141,15 @@ export function registerBrainTools(server) {
         if (!Array.isArray(records) || records.length === 0) {
             return textResult(`brain_ingest_folder: no records to ingest for slug "${safeSlug}".`);
         }
-        const targetFolder = path.join(VAULT_DIR, '04-Ingest', safeSlug);
-        mkdirSync(targetFolder, { recursive: true });
-        // Write each record to a markdown file
-        let writtenCount = 0;
-        let skippedExisting = 0;
-        for (const r of records) {
-            if (!r.content || !r.content.trim())
-                continue;
-            const fname = filenameFor(r.title, r.externalId);
-            const fullPath = path.join(targetFolder, fname);
-            const body = formatFrontmatter(r, safeSlug) + r.content;
-            // Idempotency: if a file with the same externalId already exists, overwrite
-            // (the distillation pipeline does its own content-hash dedup).
-            const preExisting = existsSync(fullPath);
-            try {
-                writeFileSync(fullPath, body, 'utf-8');
-                if (preExisting)
-                    skippedExisting += 1;
-                else
-                    writtenCount += 1;
-            }
-            catch (err) {
-                logger.warn({ err, fullPath }, 'brain_ingest_folder: write failed for one record');
-            }
-        }
-        // Run the distillation pipeline. Use a synthetic seed source so the
-        // ingestion framework can classify + distill + write back into the
-        // vault & graph with its existing dedup.
-        let ingestionSummary = '';
         try {
-            const { upsertSource, getSource } = await import('../brain/source-registry.js');
-            const { runIngestion } = await import('../brain/ingestion-pipeline.js');
-            await upsertSource({
-                slug: safeSlug,
-                kind: 'seed',
-                adapter: 'markdown',
-                configJson: JSON.stringify({ inputPath: targetFolder, managed: 'connector-feed' }),
-                targetFolder: `04-Ingest/${safeSlug}`,
-                intelligence: 'auto',
-                enabled: true,
-            });
-            const source = await getSource(safeSlug);
-            if (!source)
-                throw new Error('failed to register source');
-            const result = await runIngestion({ source, inputPath: targetFolder });
-            ingestionSummary =
-                `Pipeline: ${result.recordsIn} in · ${result.recordsWritten} written · ${result.recordsSkipped} skipped · ${result.recordsFailed} failed`;
-            if (result.errors?.length) {
-                ingestionSummary += ` (first error: ${result.errors[0].error.slice(0, 100)})`;
-            }
+            const result = await ingestBrainRecords(safeSlug, records);
+            return textResult(result.message);
         }
         catch (err) {
             const msg = err instanceof Error ? err.message : String(err);
             logger.error({ err, slug: safeSlug }, 'brain_ingest_folder: ingestion pipeline failed');
-            return textResult(`brain_ingest_folder: wrote ${writtenCount} file(s) but ingestion failed: ${msg}`);
+            return textResult(`brain_ingest_folder: ingestion failed for slug "${safeSlug}": ${msg}`);
         }
-        logger.info({ slug: safeSlug, writtenCount, skippedExisting, recordCount: records.length }, 'brain_ingest_folder complete');
-        return textResult(`Ingested into slug "${safeSlug}": ${writtenCount} new file(s), ${skippedExisting} updated in place. ${ingestionSummary}`);
     });
 }
 //# sourceMappingURL=brain-tools.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clementine-agent",
-  "version": "1.18.12",
+  "version": "1.18.14",
   "description": "Clementine — Personal AI Assistant (TypeScript)",
   "type": "module",
   "main": "dist/index.js",