@geraldmaron/construct 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/bin/construct +81 -10
- package/lib/audit-trail.mjs +77 -10
- package/lib/cli-commands.mjs +108 -1
- package/lib/demo.mjs +245 -0
- package/lib/diagram.mjs +300 -0
- package/lib/doctor/index.mjs +1 -1
- package/lib/doctor/watchers/process-pressure.mjs +1 -1
- package/lib/doctor/watchers/service-health.mjs +1 -1
- package/lib/document-extract.mjs +48 -0
- package/lib/document-ingest.mjs +7 -2
- package/lib/embed/semantic.mjs +5 -2
- package/lib/handoffs/cleanup.mjs +1 -1
- package/lib/hooks/audit-trail.mjs +14 -28
- package/lib/model-router.mjs +52 -1
- package/lib/persona-sections.mjs +66 -0
- package/lib/prompt-composer.js +2 -1
- package/lib/server/index.mjs +1 -1
- package/lib/server/langfuse-login.mjs +3 -3
- package/lib/service-manager.mjs +1 -1
- package/lib/session-store.mjs +1 -1
- package/lib/setup.mjs +39 -2
- package/lib/storage/embeddings-local.mjs +6 -3
- package/lib/storage/file-lock.mjs +18 -11
- package/package.json +4 -2
- package/personas/construct.md +11 -11
- package/scripts/sync-specialists.mjs +149 -35
- package/skills/docs/init-project.md +1 -1
- package/templates/docs/construct_guide.md +2 -2
- package/lib/ingest/chunker.mjs +0 -94
- package/lib/ingest/pipeline.mjs +0 -53
- package/lib/ingest/store.mjs +0 -82
- package/lib/mode-commands.mjs +0 -122
- package/lib/policy/unified-gates.mjs +0 -96
- package/lib/profiles/validate-custom.mjs +0 -114
- package/lib/services/telemetry-backend.mjs +0 -177
- package/lib/storage/fusion.mjs +0 -95
|
@@ -44,9 +44,11 @@ import {
|
|
|
44
44
|
writeCodexConfig,
|
|
45
45
|
} from "../lib/codex-config.mjs";
|
|
46
46
|
import { findOpenCodeConfigPath, readOpenCodeConfig, writeOpenCodeConfig } from "../lib/opencode-config.mjs";
|
|
47
|
-
import { HEAVY_EXTERNAL_MCP_IDS, LOCAL_SURFACE_MODES, decideTrim } from "../lib/mcp/tool-budget.mjs";
|
|
47
|
+
import { HEAVY_EXTERNAL_MCP_IDS, LOCAL_SURFACE_MODES, decideTrim, isLocalModel } from "../lib/mcp/tool-budget.mjs";
|
|
48
48
|
import { emitCursorRules } from "../lib/rules-delivery.mjs";
|
|
49
|
-
import { resolvePromptContract } from "../lib/prompt-composer.js";
|
|
49
|
+
import { resolvePromptContract, readPromptBody } from "../lib/prompt-composer.js";
|
|
50
|
+
import { renderPersonaForTier } from "../lib/persona-sections.mjs";
|
|
51
|
+
import { getModelVerdict } from "../lib/ollama/capability-store.mjs";
|
|
50
52
|
import {
|
|
51
53
|
buildClaudeMcpEntry,
|
|
52
54
|
buildOpenCodeMcpEntry,
|
|
@@ -56,7 +58,7 @@ import { loadConstructEnv } from "../lib/env-config.mjs";
|
|
|
56
58
|
import { inlineRoleAntiPatterns, PROMPT_WORD_CAP } from "../lib/role-preload.mjs";
|
|
57
59
|
import { loadManifest } from "../lib/roles/manifest.mjs";
|
|
58
60
|
import { resolveActiveProfile } from "../lib/profiles/loader.mjs";
|
|
59
|
-
import { resolveTiersForPrimary } from "../lib/model-router.mjs";
|
|
61
|
+
import { resolveTiersForPrimary, resolveCapabilityTier, selectLocalEditorModel } from "../lib/model-router.mjs";
|
|
60
62
|
import { stampFrontmatter } from "../lib/doc-stamp.mjs";
|
|
61
63
|
import { buildSkillFrontmatter, stripLeadingFrontmatter } from "../lib/sync/skill-frontmatter.mjs";
|
|
62
64
|
|
|
@@ -602,7 +604,83 @@ export function renderRoleFrameworkSection(entry) {
|
|
|
602
604
|
return `\n\n${lines.join("\n")}`;
|
|
603
605
|
}
|
|
604
606
|
|
|
605
|
-
|
|
607
|
+
// The native-subagent orchestration micro-prompt. A worked tool-call example lifts
|
|
608
|
+
// small local models' tool-use reliability sharply (bead construct-c16l). Shared by the
|
|
609
|
+
// full path and the capability-tiered local path so both stay in sync.
|
|
610
|
+
|
|
611
|
+
const ORCHESTRATION_MICRO_PROMPT =
|
|
612
|
+
`You are the primary orchestrator. To discover available specialist agents, you MUST call the \`orchestration_policy\` MCP tool. Do not guess agent names.\n\n` +
|
|
613
|
+
`Example — the user says "add rate limiting to the API". Your first action is a tool call, not prose:\n` +
|
|
614
|
+
` call orchestration_policy { "task": "add rate limiting to the API" }\n` +
|
|
615
|
+
`Then dispatch the specialists it returns. Always call the tool before answering.`;
|
|
616
|
+
|
|
617
|
+
// Directive for the local editor agent (construct-local). It executes bounded work on
|
|
618
|
+
// the cheap local model and hands planning/reasoning back to the construct architect —
|
|
619
|
+
// the aider architect/editor split. Kept short: a small model must actually obey it.
|
|
620
|
+
|
|
621
|
+
const LOCAL_EDITOR_DIRECTIVE =
|
|
622
|
+
`You are a focused execution agent running on a local model, dispatched by construct to do one bounded job. Do well-scoped edits for the current task and verify them; make the smallest correct change, never a broad rewrite.\n` +
|
|
623
|
+
`You do NOT plan, classify, orchestrate, or spawn other agents. For anything needing multi-file design, architecture or security judgment, dependency or contract changes, or research, STOP and return control to construct — report what needs deeper work and why, rather than attempting it yourself.`;
|
|
624
|
+
|
|
625
|
+
// Warn-and-emit capability advisory. Sizing already consumes the probe verdict
|
|
626
|
+
// (COLLAPSED → floor tier via resolveCapabilityTier); this only nudges the user toward a
|
|
627
|
+
// measured verdict and never suppresses emission. Notice-only, so it auto-suppresses in
|
|
628
|
+
// CI / test / non-TTY per the repo's wrong-context rule — no skip env var.
|
|
629
|
+
|
|
630
|
+
const localAdvisorySeen = new Set();
|
|
631
|
+
function adviseLocalModelCapability(model) {
|
|
632
|
+
if (!model || !isLocalModel(model)) return;
|
|
633
|
+
if (process.env.CI === "true" || process.env.NODE_ENV === "test" || !process.stderr.isTTY) return;
|
|
634
|
+
if (localAdvisorySeen.has(model)) return;
|
|
635
|
+
localAdvisorySeen.add(model);
|
|
636
|
+
const verdict = getModelVerdict(model)?.verdict ?? null;
|
|
637
|
+
if (verdict === "COLLAPSED") {
|
|
638
|
+
console.warn(`[sync] ${model} probed COLLAPSED — emitting at the floor tier with escalation to construct. Re-probe after a Modelfile change: construct doctor --probe-local`);
|
|
639
|
+
} else if (!verdict) {
|
|
640
|
+
console.warn(`[sync] ${model} is local with no coherence verdict — tier inferred from parameter count. For a measured tier: construct doctor --probe-local`);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
function enforcePromptWordCap(prompt, entry) {
|
|
645
|
+
const wordCount = prompt.split(/\s+/).filter(Boolean).length;
|
|
646
|
+
const effectiveCap = Number(entry.wordCapOverride) > 0 ? entry.wordCapOverride : PROMPT_WORD_CAP;
|
|
647
|
+
if (wordCount > effectiveCap) {
|
|
648
|
+
const msg = `[sync] ${entry.name}: prompt is ${wordCount} words (cap ${effectiveCap})`;
|
|
649
|
+
if (process.env.CONSTRUCT_SYNC_FORCE === '1' || process.argv.includes('--force')) {
|
|
650
|
+
console.warn(`${msg} — proceeding due to --force / CONSTRUCT_SYNC_FORCE=1.`);
|
|
651
|
+
} else {
|
|
652
|
+
console.error(`${msg}`);
|
|
653
|
+
console.error(
|
|
654
|
+
`[sync] Hard cap exceeded. Options:\n` +
|
|
655
|
+
` - trim the prompt body or move detail to a skill (preferred)\n` +
|
|
656
|
+
` - set "wordCapOverride": <N> on this entry in specialists/registry.json with a written reason\n` +
|
|
657
|
+
` - re-run with --force or CONSTRUCT_SYNC_FORCE=1 as a temporary escape hatch\n` +
|
|
658
|
+
`Prompt budget is a hard contract because every over-cap agent degrades every session that dispatches it.`,
|
|
659
|
+
);
|
|
660
|
+
process.exit(1);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
return prompt;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function buildPrompt(entry, allEntries, platform, { capabilityTier = 'full' } = {}) {
|
|
667
|
+
const capabilities = { hasNativeSubagents: HOST_KEYS.includes(platform) ? hostHasNativeSubagents(platform) : false };
|
|
668
|
+
|
|
669
|
+
// Capability-tiered local path. A small local model follows a long multi-instruction
|
|
670
|
+
// persona poorly (instruction-following degrades before the window fills), so emit
|
|
671
|
+
// only the persona sections at/below its tier plus the orchestration micro-prompt, and
|
|
672
|
+
// skip the role footer, role-framework, operating-guidance, and model-family blocks —
|
|
673
|
+
// those add instruction load the model cannot track. Cloud models resolve to 'full'
|
|
674
|
+
// and take the unchanged path below, so cloud configs are never slimmed.
|
|
675
|
+
|
|
676
|
+
if (capabilityTier && capabilityTier !== 'full' && entry.promptFile) {
|
|
677
|
+
let slim = renderPersonaForTier(readPromptBody(entry.promptFile, root), capabilityTier);
|
|
678
|
+
if (entry.injectAgentRoster && capabilities.hasNativeSubagents) {
|
|
679
|
+
slim = `${ORCHESTRATION_MICRO_PROMPT}\n\n${slim}`;
|
|
680
|
+
}
|
|
681
|
+
return enforcePromptWordCap(slim, entry);
|
|
682
|
+
}
|
|
683
|
+
|
|
606
684
|
let prompt = resolvePromptContract(entry, {
|
|
607
685
|
rootDir: root,
|
|
608
686
|
registry,
|
|
@@ -611,8 +689,6 @@ function buildPrompt(entry, allEntries, platform) {
|
|
|
611
689
|
|
|
612
690
|
prompt = inlineRoleAntiPatterns(prompt, root, entry.name, console.warn, { preload: entry.preloadRoleGuidance === true });
|
|
613
691
|
|
|
614
|
-
const capabilities = { hasNativeSubagents: HOST_KEYS.includes(platform) ? hostHasNativeSubagents(platform) : false };
|
|
615
|
-
|
|
616
692
|
// Platform-Native Orchestration Alignment (ADR-0002). Hosts with native subagent
|
|
617
693
|
// routing (OpenCode, VS Code, Cursor) do not get the static specialist roster
|
|
618
694
|
// injected — on a small-context local model the roster alone is ~3-4k tokens and,
|
|
@@ -625,14 +701,7 @@ function buildPrompt(entry, allEntries, platform) {
|
|
|
625
701
|
const roster = buildAgentRoster(allEntries);
|
|
626
702
|
prompt = `Available specialist agents:\n${roster}\n\n${prompt}`;
|
|
627
703
|
} else if (entry.injectAgentRoster && capabilities.hasNativeSubagents) {
|
|
628
|
-
|
|
629
|
-
// sharply (bead construct-c16l). Keep it to one compact turn so it stays within
|
|
630
|
-
// the prompt word cap; native-subagent hosts are exactly where local models run.
|
|
631
|
-
|
|
632
|
-
prompt = `You are the primary orchestrator. To discover available specialist agents, you MUST call the \`orchestration_policy\` MCP tool. Do not guess agent names.\n\n` +
|
|
633
|
-
`Example — the user says "add rate limiting to the API". Your first action is a tool call, not prose:\n` +
|
|
634
|
-
` call orchestration_policy { "task": "add rate limiting to the API" }\n` +
|
|
635
|
-
`Then dispatch the specialists it returns. Always call the tool before answering.\n\n${prompt}`;
|
|
704
|
+
prompt = `${ORCHESTRATION_MICRO_PROMPT}\n\n${prompt}`;
|
|
636
705
|
}
|
|
637
706
|
|
|
638
707
|
prompt += buildRoleFooter(entry);
|
|
@@ -648,26 +717,7 @@ function buildPrompt(entry, allEntries, platform) {
|
|
|
648
717
|
|
|
649
718
|
prompt += buildModelGuidanceBlock(entry);
|
|
650
719
|
|
|
651
|
-
|
|
652
|
-
const effectiveCap = Number(entry.wordCapOverride) > 0 ? entry.wordCapOverride : PROMPT_WORD_CAP;
|
|
653
|
-
if (wordCount > effectiveCap) {
|
|
654
|
-
const msg = `[sync] ${entry.name}: prompt is ${wordCount} words (cap ${effectiveCap})`;
|
|
655
|
-
if (process.env.CONSTRUCT_SYNC_FORCE === '1' || process.argv.includes('--force')) {
|
|
656
|
-
console.warn(`${msg} — proceeding due to --force / CONSTRUCT_SYNC_FORCE=1.`);
|
|
657
|
-
} else {
|
|
658
|
-
console.error(`${msg}`);
|
|
659
|
-
console.error(
|
|
660
|
-
`[sync] Hard cap exceeded. Options:\n` +
|
|
661
|
-
` - trim the prompt body or move detail to a skill (preferred)\n` +
|
|
662
|
-
` - set "wordCapOverride": <N> on this entry in specialists/registry.json with a written reason\n` +
|
|
663
|
-
` - re-run with --force or CONSTRUCT_SYNC_FORCE=1 as a temporary escape hatch\n` +
|
|
664
|
-
`Prompt budget is a hard contract because every over-cap agent degrades every session that dispatches it.`,
|
|
665
|
-
);
|
|
666
|
-
process.exit(1);
|
|
667
|
-
}
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
return prompt;
|
|
720
|
+
return enforcePromptWordCap(prompt, entry);
|
|
671
721
|
}
|
|
672
722
|
|
|
673
723
|
function escapeRegExp(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }
|
|
@@ -1638,6 +1688,20 @@ function syncOpencode(entries, targetDir = null, wants = true) {
|
|
|
1638
1688
|
}
|
|
1639
1689
|
|
|
1640
1690
|
// Write agents — no model/modelFallback set; agents inherit the global model.
|
|
1691
|
+
//
|
|
1692
|
+
// Capability tier for the orchestrator prompt. Keyed ONLY to an EXPLICIT local default
|
|
1693
|
+
// model — that is a clear intent signal we can size against at sync time. With no
|
|
1694
|
+
// explicit default (the orchestrator runs whatever model the user picks at runtime) or
|
|
1695
|
+
// a cloud default, resolveCapabilityTier returns 'full', so cloud configs and unknown
|
|
1696
|
+
// selections are never slimmed. Per-model slimming of a known pinned model lands on the
|
|
1697
|
+
// construct-local editor agent.
|
|
1698
|
+
|
|
1699
|
+
const orchestratorDefaultModel = config.model || config.defaultModel || "";
|
|
1700
|
+
adviseLocalModelCapability(orchestratorDefaultModel);
|
|
1701
|
+
const orchestratorTier = resolveCapabilityTier({
|
|
1702
|
+
model: orchestratorDefaultModel,
|
|
1703
|
+
verdict: orchestratorDefaultModel ? (getModelVerdict(orchestratorDefaultModel)?.verdict ?? null) : null,
|
|
1704
|
+
});
|
|
1641
1705
|
|
|
1642
1706
|
for (const entry of writeEntries) {
|
|
1643
1707
|
const name = adapterName(entry);
|
|
@@ -1647,7 +1711,9 @@ function syncOpencode(entries, targetDir = null, wants = true) {
|
|
|
1647
1711
|
? `${entry.role} — ${entry.description}`
|
|
1648
1712
|
: entry.description,
|
|
1649
1713
|
mode: entry.isOrchestrator ? "all" : "subagent",
|
|
1650
|
-
prompt: buildPrompt(entry, entries, "opencode"
|
|
1714
|
+
prompt: buildPrompt(entry, entries, "opencode", {
|
|
1715
|
+
capabilityTier: entry.isOrchestrator ? orchestratorTier : "full",
|
|
1716
|
+
}),
|
|
1651
1717
|
permission: {
|
|
1652
1718
|
...perms,
|
|
1653
1719
|
task: opencodeTaskPermissions(entry),
|
|
@@ -1655,6 +1721,54 @@ function syncOpencode(entries, targetDir = null, wants = true) {
|
|
|
1655
1721
|
};
|
|
1656
1722
|
}
|
|
1657
1723
|
|
|
1724
|
+
// Hybrid split (aider architect/editor). When the fast tier is a LOCAL model, emit a
|
|
1725
|
+
// narrow `construct-local` editor: it does bounded edits on a cheap local model and hands
|
|
1726
|
+
// planning/reasoning back to `construct` (the architect, which stays on the user's chosen
|
|
1727
|
+
// model — we never pin it). The editor's model is NOT the generic fast-tier default
|
|
1728
|
+
// (which for an Ollama family resolves to a non-code generalist); it is the best-installed
|
|
1729
|
+
// CODE model from this config's DECLARED local inventory (OpenCode only uses declared
|
|
1730
|
+
// models), excluding probe-COLLAPSED ones, with the fast tier as a last resort. Its prompt
|
|
1731
|
+
// is sized to the chosen model's capability tier. Deterministic name, so manage it
|
|
1732
|
+
// explicitly: emit when fast is local, delete otherwise, so switching to cloud cleans up.
|
|
1733
|
+
|
|
1734
|
+
const orchestratorEntry = writeEntries.find((e) => e.isOrchestrator) || registry.orchestrator;
|
|
1735
|
+
const orchestratorName = orchestratorEntry ? adapterName(orchestratorEntry) : "construct";
|
|
1736
|
+
const localEditorName = `${orchestratorName}-local`;
|
|
1737
|
+
if (orchestratorEntry?.promptFile && isLocalModel(resolvedModels.fast)) {
|
|
1738
|
+
const declaredLocal = Object.entries(config.provider || {})
|
|
1739
|
+
.flatMap(([pid, pv]) => Object.keys(pv?.models || {}).map((mk) => `${pid}/${mk}`))
|
|
1740
|
+
.filter((id) => isLocalModel(id) && getModelVerdict(id)?.verdict !== "COLLAPSED");
|
|
1741
|
+
const editorModel = selectLocalEditorModel(declaredLocal) || resolvedModels.fast;
|
|
1742
|
+
adviseLocalModelCapability(editorModel);
|
|
1743
|
+
const editorVerdict = getModelVerdict(editorModel)?.verdict ?? null;
|
|
1744
|
+
const editorTier = resolveCapabilityTier({ model: editorModel, verdict: editorVerdict });
|
|
1745
|
+
const editorBody = renderPersonaForTier(readPromptBody(orchestratorEntry.promptFile, root), editorTier);
|
|
1746
|
+
config.agent[localEditorName] = {
|
|
1747
|
+
description: "Local execution agent — bounded edits on the local model; escalates planning and reasoning to construct.",
|
|
1748
|
+
mode: "subagent",
|
|
1749
|
+
model: editorModel,
|
|
1750
|
+
prompt: `${LOCAL_EDITOR_DIRECTIVE}\n\n${editorBody}`,
|
|
1751
|
+
permission: {
|
|
1752
|
+
edit: "allow",
|
|
1753
|
+
bash: { "*": "allow", "rm -rf *": "deny", "git push *": "ask", "git push --force*": "ask", "git reset --hard *": "ask" },
|
|
1754
|
+
"mcp__construct-mcp__orchestration_policy": "deny",
|
|
1755
|
+
"mcp__construct-mcp__agent_contract": "deny",
|
|
1756
|
+
"mcp__construct-mcp__broker_check": "deny",
|
|
1757
|
+
"mcp__github__*": "deny",
|
|
1758
|
+
"mcp__context7__*": "deny",
|
|
1759
|
+
"mcp__sequential-thinking__*": "deny",
|
|
1760
|
+
"mcp__memory__*": "deny",
|
|
1761
|
+
// OpenCode 1.15.4 disables the `task` tool entirely for any restrictive task map
|
|
1762
|
+
// (verified in a sterile run). For an editor that is exactly right: it spawns no
|
|
1763
|
+
// subagents and escalates by RETURNING to the construct agent that dispatched it,
|
|
1764
|
+
// not by dispatching. Deny-all states that intent directly.
|
|
1765
|
+
task: { "*": "deny" },
|
|
1766
|
+
},
|
|
1767
|
+
};
|
|
1768
|
+
} else {
|
|
1769
|
+
delete config.agent[localEditorName];
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1658
1772
|
// Pass current Construct model tiers to OpenCode config for native routing.
|
|
1659
1773
|
config.construct = config.construct || {};
|
|
1660
1774
|
config.construct.models = { ...resolvedModels };
|
|
@@ -32,7 +32,7 @@ docs/ ← human-readable project documentation
|
|
|
32
32
|
1. Treat `.cx/context.md`, `.cx/context.json`, `.cx/workflow.json`, `docs/README.md`, and `docs/architecture.md` as required project state.
|
|
33
33
|
2. Read them at the start of every meaningful session.
|
|
34
34
|
3. Update them whenever work changes active reality: decisions, workflow phase, architecture assumptions, or documentation contract.
|
|
35
|
-
4. Run `construct
|
|
35
|
+
4. Run `construct dashboard` to see the project in the dashboard.
|
|
36
36
|
|
|
37
37
|
## For cx-docs-keeper
|
|
38
38
|
At session start, check the core docs set. If missing, suggest running `construct init-docs`.
|
|
@@ -109,7 +109,7 @@ All ports bind to `127.0.0.1` only; nothing is reachable from other machines on
|
|
|
109
109
|
| `construct config [mode <m>]` | Show active deployment mode (solo / team / enterprise) or set a new one |
|
|
110
110
|
| `construct doctor` | Health check across config, services, agents, hooks |
|
|
111
111
|
| `construct sync` | Regenerate platform adapters (Claude Code, OpenCode, Codex, Cursor) |
|
|
112
|
-
| `construct
|
|
112
|
+
| `construct dev` / `construct stop` | Start / stop local services |
|
|
113
113
|
| `construct status` | Live runtime status (services, providers, daemons) |
|
|
114
114
|
| `construct intake list / show / done / skip / reopen` | Drive the R&D intake queue produced from `.cx/inbox/` |
|
|
115
115
|
| `construct graph from-intake <id>` | Generate a task graph from a triaged intake packet |
|
|
@@ -131,7 +131,7 @@ That refreshes the agents, hooks, and slash commands in `.claude/` and `.constru
|
|
|
131
131
|
|
|
132
132
|
```bash
|
|
133
133
|
construct doctor # most issues surface here with a fix hint
|
|
134
|
-
construct
|
|
134
|
+
construct stop && construct dev # restart local services
|
|
135
135
|
```
|
|
136
136
|
|
|
137
137
|
Troubleshooting guide: <https://geraldmaron.github.io/construct/docs/operations/troubleshooting>
|
package/lib/ingest/chunker.mjs
DELETED
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* lib/ingest/chunker.mjs — paragraph-boundary chunker with sentence overlap.
|
|
3
|
-
*
|
|
4
|
-
* 2026-06 best practice notes:
|
|
5
|
-
* - Recursive/sentence chunking remains the high-recall default (~85-90%
|
|
6
|
-
* on 2026 RAG benchmarks). Semantic chunking gains only ~2-3% recall at
|
|
7
|
-
* ~14× the embedding cost and only matters above ~5k token docs.
|
|
8
|
-
* - Parent-document retrieval is the meaningful 2026 upgrade beyond
|
|
9
|
-
* paragraph chunking — track separately if recall plateaus.
|
|
10
|
-
*
|
|
11
|
-
* Strategy:
|
|
12
|
-
* 1. Split markdown by blank-line paragraph boundaries.
|
|
13
|
-
* 2. Pack paragraphs into chunks under maxChars (default 1500 ≈ 375 tokens).
|
|
14
|
-
* 3. Add 1-2 sentence overlap between consecutive chunks to preserve
|
|
15
|
-
* cross-chunk anchors.
|
|
16
|
-
* 4. Preserve markdown structure: never split inside a fenced code block.
|
|
17
|
-
*/
|
|
18
|
-
|
|
19
|
-
const DEFAULT_MAX_CHARS = 1500;
|
|
20
|
-
const DEFAULT_OVERLAP_SENTENCES = 2;
|
|
21
|
-
const SENTENCE_RE = /[^.!?]+[.!?]+["')\]]*\s*/g;
|
|
22
|
-
|
|
23
|
-
export function splitSentences(text) {
|
|
24
|
-
const matches = text.match(SENTENCE_RE);
|
|
25
|
-
if (matches && matches.length) return matches.map((s) => s.trim()).filter(Boolean);
|
|
26
|
-
return text.split('\n').map((s) => s.trim()).filter(Boolean);
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
function tailSentences(text, count) {
|
|
30
|
-
const sentences = splitSentences(text);
|
|
31
|
-
return sentences.slice(-count).join(' ');
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
function splitParagraphs(markdown) {
|
|
35
|
-
const blocks = [];
|
|
36
|
-
let buffer = '';
|
|
37
|
-
let inFence = false;
|
|
38
|
-
for (const line of markdown.split('\n')) {
|
|
39
|
-
if (/^```/.test(line.trim())) inFence = !inFence;
|
|
40
|
-
if (!inFence && line.trim() === '' && buffer.trim()) {
|
|
41
|
-
blocks.push(buffer.trim());
|
|
42
|
-
buffer = '';
|
|
43
|
-
continue;
|
|
44
|
-
}
|
|
45
|
-
buffer += line + '\n';
|
|
46
|
-
}
|
|
47
|
-
if (buffer.trim()) blocks.push(buffer.trim());
|
|
48
|
-
return blocks;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
export function chunkMarkdown(markdown, { maxChars = DEFAULT_MAX_CHARS, overlapSentences = DEFAULT_OVERLAP_SENTENCES } = {}) {
|
|
52
|
-
const paragraphs = splitParagraphs(markdown || '');
|
|
53
|
-
if (paragraphs.length === 0) return [];
|
|
54
|
-
|
|
55
|
-
const chunks = [];
|
|
56
|
-
let current = '';
|
|
57
|
-
let chunkIndex = 0;
|
|
58
|
-
const flushChunk = () => {
|
|
59
|
-
if (!current.trim()) return;
|
|
60
|
-
const text = current.trim();
|
|
61
|
-
chunks.push({
|
|
62
|
-
index: chunkIndex++,
|
|
63
|
-
text,
|
|
64
|
-
chars: text.length,
|
|
65
|
-
});
|
|
66
|
-
const overlap = overlapSentences > 0 ? tailSentences(text, overlapSentences) : '';
|
|
67
|
-
current = overlap ? overlap + '\n\n' : '';
|
|
68
|
-
};
|
|
69
|
-
|
|
70
|
-
for (const para of paragraphs) {
|
|
71
|
-
if (current.length + para.length + 2 > maxChars && current.trim()) {
|
|
72
|
-
flushChunk();
|
|
73
|
-
}
|
|
74
|
-
if (para.length > maxChars) {
|
|
75
|
-
flushChunk();
|
|
76
|
-
const sentences = splitSentences(para);
|
|
77
|
-
let buf = '';
|
|
78
|
-
for (const s of sentences) {
|
|
79
|
-
if (buf.length + s.length + 1 > maxChars && buf.trim()) {
|
|
80
|
-
chunks.push({ index: chunkIndex++, text: buf.trim(), chars: buf.trim().length });
|
|
81
|
-
buf = (overlapSentences > 0 ? tailSentences(buf, overlapSentences) : '') + (overlapSentences > 0 ? ' ' : '');
|
|
82
|
-
}
|
|
83
|
-
buf += s + ' ';
|
|
84
|
-
}
|
|
85
|
-
if (buf.trim()) {
|
|
86
|
-
current = buf.trim() + '\n\n';
|
|
87
|
-
}
|
|
88
|
-
continue;
|
|
89
|
-
}
|
|
90
|
-
current += (current ? '\n\n' : '') + para;
|
|
91
|
-
}
|
|
92
|
-
flushChunk();
|
|
93
|
-
return chunks;
|
|
94
|
-
}
|
package/lib/ingest/pipeline.mjs
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* lib/ingest/pipeline.mjs — orchestrate the content-hashed ingest pipeline.
|
|
3
|
-
*
|
|
4
|
-
* Flow: hash → de-dup → extract via docling/whisper → chunk → write to
|
|
5
|
-
* .cx/ingest/<sha>/. Returns a record describing what was stored along
|
|
6
|
-
* with any droppedInfo surfaced by the extractor.
|
|
7
|
-
*
|
|
8
|
-
* Idempotent: re-ingesting the same content (by sha256) returns the
|
|
9
|
-
* existing record without re-extracting.
|
|
10
|
-
*/
|
|
11
|
-
import fs from 'node:fs/promises';
|
|
12
|
-
import path from 'node:path';
|
|
13
|
-
import { extractDocumentTextAsync } from '../document-extract.mjs';
|
|
14
|
-
import { hashFile, defaultIngestRoot, readRecord, writeRecord } from './store.mjs';
|
|
15
|
-
import { chunkMarkdown } from './chunker.mjs';
|
|
16
|
-
|
|
17
|
-
export async function ingestFile(filePath, { cwd = process.cwd(), force = false } = {}) {
|
|
18
|
-
const absPath = path.resolve(filePath);
|
|
19
|
-
const stat = await fs.stat(absPath);
|
|
20
|
-
if (!stat.isFile()) throw new Error(`not a regular file: ${absPath}`);
|
|
21
|
-
|
|
22
|
-
const root = defaultIngestRoot(cwd);
|
|
23
|
-
const sha256 = await hashFile(absPath);
|
|
24
|
-
|
|
25
|
-
if (!force) {
|
|
26
|
-
const existing = await readRecord(root, sha256);
|
|
27
|
-
if (existing) return { ...existing, status: 'cached' };
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
const extracted = await extractDocumentTextAsync(absPath);
|
|
31
|
-
const markdown = extracted.markdown ?? extracted.text ?? '';
|
|
32
|
-
const chunks = chunkMarkdown(markdown);
|
|
33
|
-
|
|
34
|
-
const source = {
|
|
35
|
-
sourcePath: absPath,
|
|
36
|
-
fileName: path.basename(absPath),
|
|
37
|
-
extension: extracted.extension,
|
|
38
|
-
bytes: stat.size,
|
|
39
|
-
sha256,
|
|
40
|
-
ingestedAt: new Date().toISOString(),
|
|
41
|
-
};
|
|
42
|
-
|
|
43
|
-
const meta = {
|
|
44
|
-
extractionMethod: extracted.extractionMethod,
|
|
45
|
-
extractorMetadata: extracted.metadata ?? null,
|
|
46
|
-
droppedInfo: extracted.droppedInfo ?? [],
|
|
47
|
-
chunkCount: chunks.length,
|
|
48
|
-
chunkChars: chunks.reduce((a, c) => a + c.chars, 0),
|
|
49
|
-
};
|
|
50
|
-
|
|
51
|
-
const record = await writeRecord(root, { sha256, source, meta, markdown });
|
|
52
|
-
return { ...record, status: 'ingested', droppedInfo: meta.droppedInfo, chunks };
|
|
53
|
-
}
|
package/lib/ingest/store.mjs
DELETED
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* lib/ingest/store.mjs — content-addressed ingest store at .cx/ingest/<hash>/.
|
|
3
|
-
*
|
|
4
|
-
* Each ingested document is stored under its SHA-256 content hash:
|
|
5
|
-
* .cx/ingest/<sha256>/source.json — original path, size, sha256, ingestedAt
|
|
6
|
-
* .cx/ingest/<sha256>/markdown.md — extracted markdown body
|
|
7
|
-
* .cx/ingest/<sha256>/meta.json — extractor metadata, droppedInfo, chunks
|
|
8
|
-
*
|
|
9
|
-
* Idempotent: re-ingesting a file with the same content is a no-op (returns
|
|
10
|
-
* the existing record). Re-running ingestion is the safe default.
|
|
11
|
-
*
|
|
12
|
-
* 2026-06 best practice notes:
|
|
13
|
-
* - SHA-256 is still the universal content-addressing primitive. BLAKE3 is
|
|
14
|
-
* ~2.3× faster on M-series and worth considering at multi-GB scale,
|
|
15
|
-
* but is not the industry default and adds an unfamiliar dependency.
|
|
16
|
-
* Revisit if ingest throughput becomes a measured bottleneck.
|
|
17
|
-
* - Storing markdown body separately from metadata keeps the embedding
|
|
18
|
-
* and search paths simple: they only read markdown.md and look up
|
|
19
|
-
* provenance from meta.json on demand.
|
|
20
|
-
*/
|
|
21
|
-
import { createHash } from 'node:crypto';
|
|
22
|
-
import { createReadStream } from 'node:fs';
|
|
23
|
-
import fs from 'node:fs/promises';
|
|
24
|
-
import path from 'node:path';
|
|
25
|
-
|
|
26
|
-
export function defaultIngestRoot(cwd = process.cwd()) {
|
|
27
|
-
return path.join(cwd, '.cx', 'ingest');
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export async function hashFile(filePath) {
|
|
31
|
-
return new Promise((resolveHash, reject) => {
|
|
32
|
-
const hash = createHash('sha256');
|
|
33
|
-
const stream = createReadStream(filePath);
|
|
34
|
-
stream.on('data', (chunk) => hash.update(chunk));
|
|
35
|
-
stream.on('error', reject);
|
|
36
|
-
stream.on('end', () => resolveHash(hash.digest('hex')));
|
|
37
|
-
});
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
function recordDirFor(root, sha256) {
|
|
41
|
-
return path.join(root, sha256);
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
export async function readRecord(root, sha256) {
|
|
45
|
-
const dir = recordDirFor(root, sha256);
|
|
46
|
-
try {
|
|
47
|
-
const [source, meta, markdown] = await Promise.all([
|
|
48
|
-
fs.readFile(path.join(dir, 'source.json'), 'utf8').then(JSON.parse),
|
|
49
|
-
fs.readFile(path.join(dir, 'meta.json'), 'utf8').then(JSON.parse),
|
|
50
|
-
fs.readFile(path.join(dir, 'markdown.md'), 'utf8'),
|
|
51
|
-
]);
|
|
52
|
-
return { sha256, source, meta, markdown, dir };
|
|
53
|
-
} catch {
|
|
54
|
-
return null;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
export async function writeRecord(root, { sha256, source, meta, markdown }) {
|
|
59
|
-
const dir = recordDirFor(root, sha256);
|
|
60
|
-
await fs.mkdir(dir, { recursive: true });
|
|
61
|
-
await Promise.all([
|
|
62
|
-
fs.writeFile(path.join(dir, 'source.json'), JSON.stringify(source, null, 2) + '\n', 'utf8'),
|
|
63
|
-
fs.writeFile(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n', 'utf8'),
|
|
64
|
-
fs.writeFile(path.join(dir, 'markdown.md'), markdown, 'utf8'),
|
|
65
|
-
]);
|
|
66
|
-
return { sha256, dir, source, meta, markdown };
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
export async function listRecords(root) {
|
|
70
|
-
let entries;
|
|
71
|
-
try { entries = await fs.readdir(root, { withFileTypes: true }); }
|
|
72
|
-
catch { return []; }
|
|
73
|
-
const records = [];
|
|
74
|
-
for (const entry of entries) {
|
|
75
|
-
if (!entry.isDirectory()) continue;
|
|
76
|
-
const sha = entry.name;
|
|
77
|
-
if (!/^[a-f0-9]{64}$/.test(sha)) continue;
|
|
78
|
-
const record = await readRecord(root, sha);
|
|
79
|
-
if (record) records.push(record);
|
|
80
|
-
}
|
|
81
|
-
return records;
|
|
82
|
-
}
|
package/lib/mode-commands.mjs
DELETED
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* lib/mode-commands.mjs — Standardized command contracts across deployment modes.
|
|
3
|
-
*
|
|
4
|
-
* Provides mode-aware command execution for intake, memory, and workflow operations.
|
|
5
|
-
* Routes to file-based implementations in solo mode, Postgres in team/enterprise mode.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
// lib/mode-commands.mjs
|
|
9
|
-
// Standardized command contracts across deployment modes
|
|
10
|
-
|
|
11
|
-
import { spawnSync } from 'node:child_process';
|
|
12
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
13
|
-
import { basename, join } from 'node:path';
|
|
14
|
-
|
|
15
|
-
export const MODE_COMMANDS = {
|
|
16
|
-
// Intake queue operations
|
|
17
|
-
intake: {
|
|
18
|
-
list: async (options = {}) => {
|
|
19
|
-
const mode = getDeploymentMode();
|
|
20
|
-
|
|
21
|
-
if (mode === 'solo') {
|
|
22
|
-
// File-based intake
|
|
23
|
-
const result = spawnSync('find', ['.cx/intake/pending', '-name', '*.json', '-type', 'f'],
|
|
24
|
-
{ encoding: 'utf8', cwd: options.cwd });
|
|
25
|
-
return parseIntakeFiles(result.stdout);
|
|
26
|
-
} else {
|
|
27
|
-
// Postgres-based intake
|
|
28
|
-
return queryPostgresIntake(options);
|
|
29
|
-
}
|
|
30
|
-
},
|
|
31
|
-
|
|
32
|
-
show: async (id, options = {}) => {
|
|
33
|
-
const mode = getDeploymentMode();
|
|
34
|
-
|
|
35
|
-
if (mode === 'solo') {
|
|
36
|
-
return readIntakeFile(id, options);
|
|
37
|
-
} else {
|
|
38
|
-
return queryPostgresIntakeItem(id, options);
|
|
39
|
-
}
|
|
40
|
-
},
|
|
41
|
-
},
|
|
42
|
-
|
|
43
|
-
// Storage operations
|
|
44
|
-
storage: {
|
|
45
|
-
sync: async (options = {}) => {
|
|
46
|
-
const mode = getDeploymentMode();
|
|
47
|
-
|
|
48
|
-
// Same interface, different implementation
|
|
49
|
-
if (mode === 'solo') {
|
|
50
|
-
return syncToFileStorage(options);
|
|
51
|
-
} else {
|
|
52
|
-
return syncToPostgresStorage(options);
|
|
53
|
-
}
|
|
54
|
-
},
|
|
55
|
-
|
|
56
|
-
query: async (criteria, options = {}) => {
|
|
57
|
-
// Unified query interface
|
|
58
|
-
return queryStorage(criteria, options);
|
|
59
|
-
},
|
|
60
|
-
},
|
|
61
|
-
};
|
|
62
|
-
|
|
63
|
-
function getDeploymentMode() {
|
|
64
|
-
return process.env.CONSTRUCT_DEPLOYMENT_MODE || 'solo';
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
function parseIntakeFiles(stdout) {
|
|
68
|
-
return stdout.split('\n')
|
|
69
|
-
.filter(Boolean)
|
|
70
|
-
.map(f => ({ id: basename(f, '.json'), source: 'file' }));
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
function readIntakeFile(id, options) {
|
|
74
|
-
const filePath = join(options.cwd || process.cwd(), '.cx/intake/pending', `${id}.json`);
|
|
75
|
-
if (!existsSync(filePath)) {
|
|
76
|
-
return { error: `Intake item ${id} not found` };
|
|
77
|
-
}
|
|
78
|
-
return JSON.parse(readFileSync(filePath, 'utf8'));
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
async function queryPostgresIntake(options) {
|
|
82
|
-
// Would use SQL client
|
|
83
|
-
return { items: [], mode: 'postgres' };
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
async function queryPostgresIntakeItem(id, options) {
|
|
87
|
-
return { id, mode: 'postgres' };
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
async function syncToFileStorage(options) {
|
|
91
|
-
return {
|
|
92
|
-
success: true,
|
|
93
|
-
mode: 'solo',
|
|
94
|
-
backend: 'file',
|
|
95
|
-
};
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
async function syncToPostgresStorage(options) {
|
|
99
|
-
return {
|
|
100
|
-
success: true,
|
|
101
|
-
mode: 'team',
|
|
102
|
-
backend: 'postgres',
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
async function queryStorage(criteria, options) {
|
|
107
|
-
// Unified query - works across modes
|
|
108
|
-
return {
|
|
109
|
-
results: [],
|
|
110
|
-
mode: getDeploymentMode(),
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
export function getCommandInterface(command, operation) {
|
|
115
|
-
const cmd = MODE_COMMANDS[command];
|
|
116
|
-
if (!cmd) return null;
|
|
117
|
-
|
|
118
|
-
const fn = cmd[operation];
|
|
119
|
-
if (!fn) return null;
|
|
120
|
-
|
|
121
|
-
return fn;
|
|
122
|
-
}
|