@geraldmaron/construct 1.0.17 → 1.0.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/README.md +6 -4
  2. package/bin/construct +26 -3
  3. package/commands/design/flow.md +2 -0
  4. package/commands/design/ui.md +2 -0
  5. package/db/schema/003_observation_reconciliation.sql +14 -0
  6. package/lib/bootstrap/resources.mjs +0 -1
  7. package/lib/cli-commands.mjs +33 -5
  8. package/lib/comment-lint.mjs +44 -0
  9. package/lib/contracts/validate.mjs +106 -0
  10. package/lib/decisions/enforced-baseline.json +23 -0
  11. package/lib/decisions/golden.mjs +87 -0
  12. package/lib/decisions/precedence.mjs +46 -0
  13. package/lib/decisions/registry.mjs +469 -0
  14. package/lib/deployment/parity-contract.mjs +148 -0
  15. package/lib/embed/cli.mjs +11 -0
  16. package/lib/embed/conflict-detection.mjs +4 -4
  17. package/lib/embed/customer-profiles.mjs +1 -1
  18. package/lib/embed/reconcile.mjs +60 -0
  19. package/lib/embed/semantic.mjs +5 -3
  20. package/lib/gates-audit.mjs +2 -2
  21. package/lib/hooks/config-protection.mjs +22 -3
  22. package/lib/hooks/guard-bash.mjs +1 -1
  23. package/lib/init-docs.mjs +1 -0
  24. package/lib/mode-commands.mjs +6 -8
  25. package/lib/observation-store.mjs +16 -2
  26. package/lib/opencode-telemetry.mjs +1 -1
  27. package/lib/roles/cli.mjs +10 -2
  28. package/lib/roles/gateway.mjs +50 -1
  29. package/lib/scheduler/index.mjs +31 -0
  30. package/lib/server/index.mjs +13 -3
  31. package/lib/server/static/index.html +1 -1
  32. package/lib/setup.mjs +6 -0
  33. package/lib/storage/hybrid-query.mjs +49 -38
  34. package/lib/storage/rrf.mjs +42 -0
  35. package/lib/storage/vector-client.mjs +18 -3
  36. package/lib/telemetry/backends/local.mjs +1 -1
  37. package/lib/telemetry/skill-calls.mjs +1 -1
  38. package/lib/templates/visual-requirements.mjs +84 -0
  39. package/package.json +12 -5
  40. package/rules/common/comments.md +3 -0
  41. package/rules/common/no-fabrication.md +3 -0
  42. package/rules/common/precedence.md +19 -0
  43. package/rules/common/research-sources.md +32 -0
  44. package/rules/common/research.md +59 -2
  45. package/skills/roles/data-engineer.pipeline.md +13 -1
  46. package/skills/roles/debugger.md +9 -0
  47. package/skills/roles/designer.accessibility.md +13 -3
  48. package/skills/roles/designer.md +10 -0
  49. package/skills/roles/engineer.platform.md +8 -0
  50. package/skills/roles/operator.md +10 -1
  51. package/skills/roles/operator.release.md +8 -0
  52. package/skills/roles/operator.sre.md +10 -1
  53. package/skills/roles/orchestrator.md +9 -2
  54. package/skills/roles/product-manager.business-strategy.md +10 -1
  55. package/skills/roles/researcher.explorer.md +12 -1
  56. package/skills/roles/researcher.ux.md +13 -1
  57. package/skills/roles/reviewer.devil-advocate.md +14 -2
  58. package/skills/roles/reviewer.evaluator.md +17 -4
  59. package/skills/roles/reviewer.trace.md +12 -1
  60. package/skills/roles/security.legal-compliance.md +8 -0
  61. package/skills/roles/security.md +11 -0
  62. package/specialists/contracts.json +18 -0
  63. package/specialists/prompts/cx-architect.md +1 -0
  64. package/specialists/prompts/cx-designer.md +1 -1
  65. package/specialists/prompts/cx-operations.md +1 -0
  66. package/specialists/prompts/cx-rd-lead.md +1 -0
  67. package/specialists/prompts/cx-researcher.md +4 -2
  68. package/specialists/prompts/cx-sre.md +1 -0
  69. package/specialists/registry.json +2 -2
  70. package/templates/docs/backlog-proposal.md +1 -1
  71. package/templates/docs/customer-profile.md +1 -1
  72. package/templates/docs/evidence-brief.md +5 -1
  73. package/templates/docs/incident-report.md +37 -21
  74. package/templates/docs/prfaq.md +2 -2
  75. package/templates/docs/product-intelligence-report.md +1 -1
  76. package/templates/docs/research-brief.md +8 -6
  77. package/templates/docs/research-finding.md +32 -7
  78. package/templates/docs/rfc.md +13 -1
  79. package/templates/docs/runbook.md +20 -1
  80. package/templates/docs/signal-brief.md +4 -1
  81. package/templates/docs/skill-artifact.md +27 -7
  82. package/templates/docs/strategy.md +23 -2
  83. package/templates/docs/test-plan.md +96 -0
  84. package/lib/bootstrap/lazy-install.mjs +0 -161
  85. package/lib/embed/jobs/vector-sync.mjs +0 -198
  86. package/lib/knowledge/postgres-search.mjs +0 -132
  87. package/lib/services/pattern-promotion-service.mjs +0 -167
  88. package/lib/storage/unified-storage.mjs +0 -550
@@ -1,26 +1,51 @@
1
1
  ---
2
2
  kind: research-finding
3
3
  topic: "<short topic line>"
4
- confidence: inferred
4
+ confidence: high | medium | low
5
5
  sources: []
6
6
  created: <ISO timestamp set by construct knowledge add>
7
- expiresAt: <ISO timestamp, default created + 90d>
7
+ expiresAt: <ISO timestamp, default created + 90d; shorten for fast-moving topics>
8
8
  profile: <active profile id>
9
9
  ---
10
10
 
11
+ <!--
12
+ A lightweight but rigorous knowledge-store entry. Same evidence standard as a
13
+ research brief, condensed: observation separated from inference, every finding
14
+ cited, confidence tied to source grade, an explicit refresh trigger. See
15
+ rules/common/research.md.
16
+ -->
17
+
18
+ ## SOURCES
19
+
20
+ | Title / Path | Class | Reliability | Credibility | Date | URL | Verified |
21
+ |---|---|---|---|---|---|---|
22
+ | {source} | primary / secondary / tertiary | A–F | 1–6 | {YYYY-MM-DD} | {url} | yes / no |
23
+
11
24
  ## FINDINGS
12
25
 
13
- - <Finding 1 with citation>
14
- - <Finding 2 with citation>
26
+ <!-- Observation only: what the source states. Each finding cites a source row above. -->
27
+
28
+ - <Finding 1 [source: …]>
29
+ - <Finding 2 [source: …]>
15
30
 
16
31
  ## INFERENCES
17
32
 
18
- - <Inference labeled as such>
33
+ <!-- What is concluded beyond what any single source says. Labeled as inference, not fact. -->
34
+
35
+ - <Inference, with the findings it rests on>
36
+
37
+ ## CONFIDENCE
38
+
39
+ <!-- high / medium / low, with the reasoning: tie it to the Admiralty grade of the sources (research.md §10). high only on A1/A2/B1. Name the strongest counter-evidence. -->
19
40
 
20
41
  ## GAPS
21
42
 
22
- - <What we could not confirm>
43
+ - <What could not be confirmed and would change the conclusion if known>
23
44
 
24
45
  ## RECOMMENDATION
25
46
 
26
- - <Next action based on the evidence available>
47
+ - <Next action based on the evidence available, and the threshold that would change it>
48
+
49
+ ## REFRESH
50
+
51
+ <!-- When this finding should be re-verified: the expiresAt date, or the event that would invalidate it (a new release, a superseding paper). -->
@@ -22,7 +22,19 @@ contract, API, schema, or protocol consumed by other systems.
22
22
  <!-- What problem or limitation drives this proposal? Include evidence: incidents, performance data, support tickets, developer friction. Explain why the status quo is not acceptable. -->
23
23
 
24
24
  ## Proposed design
25
- <!-- The concrete proposal. Enough detail for reviewers to evaluate feasibility and tradeoffs. Diagrams, code sketches, and data models are appropriate here. -->
25
+ <!-- The concrete proposal. Enough detail for reviewers to evaluate feasibility and tradeoffs. The sequence below shows the proposed behavior in context — include the happy path and at least one error path. -->
26
+
27
+ ```mermaid
28
+ sequenceDiagram
29
+ participant Client
30
+ participant Service
31
+ participant Store
32
+ Client->>Service: request
33
+ Service->>Store: read or write
34
+ Store-->>Service: result
35
+ Service-->>Client: response
36
+ Note over Service,Store: error path — Store unavailable → Service returns degraded response
37
+ ```
26
38
 
27
39
  ## Tradeoffs and alternatives
28
40
  <!-- The other credible designs considered. For each: what it is, why it was not chosen, and under what conditions it would be preferred. No strawmen. -->
@@ -14,8 +14,27 @@
14
14
  ## Impact
15
15
  <!-- Who is affected and how badly. Data loss? Degraded performance? Complete outage? -->
16
16
 
17
+ ## Severity and response
18
+ <!-- Map each severity to the response it triggers: page urgency, comms cadence, and error-budget consequence. A SEV-1 pages immediately and freezes related releases per the error-budget policy (https://sre.google/workbook/error-budget-policy/); a SEV-3 is handled in business hours. -->
19
+
20
+ | Severity | Trigger condition | Page within | Comms | Error budget |
21
+ |----------|-------------------|-------------|-------|--------------|
22
+ | SEV-1 | {full outage / data loss / SLO breach} | 5 min | exec + status page | breach → freeze releases |
23
+ | SEV-2 | {major degradation, workaround exists} | 15 min | team + stakeholders | partial spend |
24
+ | SEV-3 | {minor / single-tenant / cosmetic} | business hours | team channel | none |
25
+
17
26
  ## Diagnostic steps
18
- <!-- Ordered checks from cheapest to most expensive. Each step: what to check, how to check it, what the answer means. -->
27
+ <!-- Ordered checks from cheapest to most expensive. Each step: what to check, how to check it, what the answer means. Keep the decision tree below in sync with the steps. -->
28
+
29
+ ```mermaid
30
+ flowchart TD
31
+ A[Alert fires] --> B{Error rate elevated?}
32
+ B -->|Yes| C[Check upstream dependency health]
33
+ B -->|No| D{User-visible impact?}
34
+ C --> E[Apply remediation]
35
+ D -->|Yes| C
36
+ D -->|No| F[Monitor and stand down]
37
+ ```
19
38
 
20
39
  ## Remediation
21
40
  <!-- The fix. Step-by-step, with exact commands or UI paths. Include expected output for each step. -->
@@ -13,7 +13,10 @@ without pretending the requirements are ready.
13
13
  <!-- One or two paragraphs. What appears to be happening, and where the signal came from. -->
14
14
 
15
15
  ## Evidence
16
- <!-- Cite the concrete notes, tickets, customer quotes, traces, or analytics that support the signal. -->
16
+ <!-- Cite the concrete notes, tickets, customer quotes, traces, or analytics. Grade each source with its Admiralty reliability/credibility (rules/common/research.md §10); for community sources note corroboration and engagement. The signal strength above should follow from the grades, not precede them. -->
17
+
18
+ ## Counter-signal
19
+ <!-- The strongest evidence against the signal, or its absence. A signal no one tried to disconfirm is weaker than it looks. -->
17
20
 
18
21
  ## Why it matters
19
22
  <!-- What product, customer, or business decision this could affect if the signal strengthens. -->
@@ -1,9 +1,11 @@
1
1
  ---
2
2
  name: <verb-led-id>
3
3
  scope: "<one paragraph: what it covers, what it does not>"
4
- observable_outcome: <how someone outside the role tells this skill happened>
4
+ observable_outcome: <how someone outside the role tells this skill happened — a concrete artifact, not "understands X">
5
+ prerequisites:
6
+ - <skill-id or capability that must exist before this one is useful>
5
7
  provenance:
6
- - <citation: post-mortem, public framework, competency model>
8
+ - <citation: post-mortem, public framework, competency model, with URL>
7
9
  roles:
8
10
  - <role-id that uses this skill>
9
11
  ---
@@ -12,16 +14,34 @@ roles:
12
14
 
13
15
  ## What this skill produces
14
16
 
15
- <Concrete output. Bloom-style: not "knows X" but "produces Y".>
17
+ <Concrete output, Bloom-style: not "knows X" but "produces Y". Name the artifact and what makes it correct. Example: "a runbook whose diagnostic flowchart branches to a fix in ≤3 steps", not "understands incident response".>
16
18
 
17
19
  ## When to invoke it
18
20
 
19
- <Triggers. What the operator or the upstream persona is doing when this skill is the right call.>
21
+ <Triggers. What the operator or upstream persona is doing when this skill is the right call. Include the signal that distinguishes it from adjacent skills.>
20
22
 
21
- ## How it composes
23
+ ## Prerequisites and composition
22
24
 
23
- <Other skills typically chained before or after.>
25
+ <What must be true or done first, and the skills typically chained before and after. A skill with unmet prerequisites produces confident-looking but wrong output.>
26
+
27
+ ## Competency rubric
28
+
29
+ <What the skill looks like at each level. The author of an artifact should be able to place their work on this scale.>
30
+
31
+ | Level | What it looks like |
32
+ |-------|--------------------|
33
+ | Novice | <produces the artifact but misses edge cases / evidence / failure paths> |
34
+ | Competent | <covers the happy path and the common failure modes; cites sources> |
35
+ | Expert | <anticipates the non-obvious failure, calibrates confidence, leaves the next person a reproducible trail> |
36
+
37
+ ## Failure modes
38
+
39
+ <How this skill breaks when misapplied — distinct from anti-patterns. What does a wrong-but-plausible output look like, and what signal reveals it? When should the operator escalate or stop?>
24
40
 
25
41
  ## Anti-patterns
26
42
 
27
- <Ways this skill gets misapplied; what looks similar but is not this.>
43
+ <Ways this skill gets misapplied; what looks similar but is not this skill.>
44
+
45
+ ## Worked example
46
+
47
+ <A short, concrete instance: the trigger, the artifact produced, and why it meets the observable outcome. One real example removes more ambiguity than a page of guidance.>
@@ -15,8 +15,8 @@ Owning workflow: skills/docs/strategy-workflow.md.
15
15
  <!-- 1–3 sentences. Where are we going in 3–5 years? -->
16
16
 
17
17
  ## Bets
18
- <!-- Explicit choices. Each bet: what we're doing and why. -->
19
- - **Bet 1**: …
18
+ <!-- Explicit choices. Each bet: what we're doing, why, the leading indicator that tells us early if it's working, and the kill criterion that ends it. A bet with no kill criterion is a belief, not a bet. -->
19
+ - **Bet 1**: … **Why**: … **Leading indicator**: … **Kill criterion**: {what observation would make us stop}
20
20
  - **Bet 2**: …
21
21
 
22
22
  ## Non-bets
@@ -29,6 +29,27 @@ Owning workflow: skills/docs/strategy-workflow.md.
29
29
  ## North Star Metric
30
30
  <!-- Single measurable outcome. Format: Metric / Baseline / Target / Owner -->
31
31
 
32
+ ## Metrics
33
+ <!-- The North Star is lagging. Name the leading indicators that move first and predict it, so a failing bet is visible before the lagging metric confirms it. -->
34
+
35
+ | Metric | Type (leading/lagging) | Baseline | Target | Owner |
36
+ |--------|------------------------|----------|--------|-------|
37
+ | {name} | leading / lagging | {value} | {value} | {who} |
38
+
39
+ ## Milestones
40
+ <!-- What must be true by when. Each milestone is a checkpoint that confirms or kills a bet. -->
41
+
42
+ | Milestone | By when | Confirms / kills |
43
+ |-----------|---------|------------------|
44
+ | {what} | {date} | {which bet} |
45
+
46
+ ## Risks
47
+ <!-- What could invalidate the strategy. Risk register, not a worry list. -->
48
+
49
+ | Risk | Likelihood | Impact | Mitigation / leading signal |
50
+ |------|-----------|--------|------------------------------|
51
+ | {what} | high/med/low | high/med/low | {response} |
52
+
32
53
  ## Competitive Positioning
33
54
  <!-- How we differentiate. 2–4 sentences. -->
34
55
 
@@ -0,0 +1,96 @@
1
+ # Test Plan: {feature or system name}
2
+
3
+ - **Date**: {YYYY-MM-DD}
4
+ - **Owner**: {name or role}
5
+ - **Status**: draft | in-review | approved | executed
6
+ - **Related**: {PRD / ADR paths or none}
7
+
8
+ <!--
9
+ Use this when a feature, migration, or system change needs a deliberate
10
+ verification strategy before it ships. The plan is the contract between
11
+ what was specified and what will be proven; every scenario below must
12
+ trace to a requirement or an observed risk, never to guesswork.
13
+
14
+ Before drafting, read rules/common/framing.md and rules/common/no-fabrication.md.
15
+ Acceptance criteria must be binary pass/fail. If a criterion cannot be
16
+ checked without asking the author, it is not done. Cite the requirement
17
+ (FR/NFR id, ADR decision, ticket) each scenario verifies.
18
+ -->
19
+
20
+ ## Scope
21
+ <!--
22
+ What is under test and what is explicitly not. Name the components, surfaces,
23
+ or behaviors covered. The out-of-scope list protects reviewer attention and
24
+ makes coverage gaps deliberate rather than accidental.
25
+ -->
26
+
27
+ | | Description |
28
+ |---|---|
29
+ | **In scope** | <what this plan verifies> |
30
+ | **Out of scope** | <related behavior deliberately not tested here, and why> |
31
+
32
+ ## Test strategy
33
+ <!--
34
+ The levels of testing applied and why each is justified. Map levels to risk:
35
+ unit for logic, integration for contracts and boundaries, end-to-end for
36
+ user-visible flows, manual/exploratory for what automation cannot reach.
37
+ State what each level is responsible for so coverage gaps are visible.
38
+ -->
39
+
40
+ | Level | What it covers | Why it lives at this level |
41
+ |---|---|---|
42
+ | Unit | <logic, pure functions, edge cases> | <reason> |
43
+ | Integration | <contracts, boundaries, data access> | <reason> |
44
+ | End-to-end | <user-visible flows> | <reason> |
45
+ | Manual / exploratory | <what automation cannot reach> | <reason> |
46
+
47
+ ## Key scenarios
48
+ <!--
49
+ The scenarios that prove the change works. Each scenario names the requirement
50
+ it verifies, the setup, the action, and the binary acceptance criterion. A
51
+ reviewer must be able to run or read the scenario and decide pass/fail without
52
+ asking the author.
53
+ -->
54
+
55
+ | ID | Scenario | Verifies | Acceptance (binary pass/fail) |
56
+ |---|---|---|---|
57
+ | TC-1 | <setup, action, expected result> | <FR/NFR id or risk> | <observable condition> |
58
+ | TC-2 | <...> | <...> | <...> |
59
+
60
+ ## Risks and edge cases
61
+ <!--
62
+ The failure modes and boundary conditions worth explicit attention: empty
63
+ inputs, concurrency, partial failure, large data, permission boundaries,
64
+ rollback. For each, state how the plan covers it or why the residual risk
65
+ is accepted.
66
+ -->
67
+
68
+ | Risk / edge case | Likelihood | Impact | Coverage or accepted reason |
69
+ |---|---|---|---|
70
+ | <condition> | low / med / high | low / med / high | <how it is tested, or why accepted> |
71
+
72
+ ## Environments and data
73
+ <!--
74
+ Where tests run and what data they use. Name the environments, fixtures,
75
+ seed data, and any external dependencies (sandboxes, stubs, recorded
76
+ responses). Flag data that must be synthetic or anonymized.
77
+ -->
78
+
79
+ ## Entry and exit criteria
80
+ <!--
81
+ Entry: what must be true before testing starts (build available, environment
82
+ provisioned, dependencies stubbed). Exit: what must be true to call testing
83
+ done (all blocking scenarios pass, known issues triaged, coverage threshold
84
+ met). Both are checklists a reviewer can verify.
85
+ -->
86
+
87
+ **Entry criteria**
88
+
89
+ - <condition that must hold before execution begins>
90
+
91
+ **Exit criteria**
92
+
93
+ - <condition that must hold to declare the plan executed>
94
+
95
+ ## References
96
+ <!-- PRD, ADR, RFC, related test plans, prior incidents. -->
@@ -1,161 +0,0 @@
1
- /**
2
- * lib/bootstrap/lazy-install.mjs — on-demand resource installer with consent gating.
3
- *
4
- * When a runtime code path needs a resource that is not present, call
5
- * `lazyInstall(id)`. The function:
6
- *
7
- * 1. Reads cached operator consent from config.env (consent key set by the
8
- * resource definition in `lib/bootstrap/resources.mjs`).
9
- * 2. If consent is 'never', returns { installed: false, fallback: <text> }
10
- * without prompting or printing anything.
11
- * 3. If a TTY is available, prompts the user with the resource displayName,
12
- * estimated download size, and [y/n/never].
13
- * 4. On 'y' or an existing 'yes' consent: calls resource.install(), waits
14
- * for it to complete, updates config.env, and returns { installed: true }.
15
- * 5. On 'n': returns { installed: false, fallback: <text> } without saving.
16
- * 6. On 'never': writes 'never' to config.env and returns { installed: false }.
17
- *
18
- * Hooks that run without a TTY (most hook invocations) skip to step 2 and
19
- * use the fallback silently.
20
- *
21
- * Consent is cached per-resource in config.env so the prompt never repeats
22
- * across process restarts once the user has answered.
23
- */
24
-
25
- import fs from 'node:fs';
26
- import path from 'node:path';
27
- import os from 'node:os';
28
- import readline from 'node:readline';
29
-
30
- import { getResource, probeResource } from './resources.mjs';
31
-
32
- const CONFIG_ENV = path.join(os.homedir(), '.construct', 'config.env');
33
-
34
- function readConfigEnv() {
35
- try {
36
- return fs.readFileSync(CONFIG_ENV, 'utf8');
37
- } catch {
38
- return '';
39
- }
40
- }
41
-
42
- function readConsentFromEnv(consentKey) {
43
- const lines = readConfigEnv().split('\n');
44
- for (const line of lines) {
45
- const trimmed = line.trim();
46
- if (!trimmed || trimmed.startsWith('#')) continue;
47
- const eq = trimmed.indexOf('=');
48
- if (eq === -1) continue;
49
- const k = trimmed.slice(0, eq).trim();
50
- const v = trimmed.slice(eq + 1).trim().replace(/^['"]|['"]$/g, '');
51
- if (k === consentKey) return v;
52
- }
53
- return null;
54
- }
55
-
56
- function writeConsentToEnv(consentKey, value) {
57
- let content = readConfigEnv();
58
- const regex = new RegExp(`^${consentKey}=.*$`, 'm');
59
- const line = `${consentKey}=${value}`;
60
- if (regex.test(content)) {
61
- content = content.replace(regex, line);
62
- } else {
63
- content = content.trimEnd() + (content ? '\n' : '') + line + '\n';
64
- }
65
- fs.mkdirSync(path.dirname(CONFIG_ENV), { recursive: true });
66
- fs.writeFileSync(CONFIG_ENV, content, { mode: 0o600 });
67
- }
68
-
69
- function hasTty() {
70
- return Boolean(process.stdin.isTTY && process.stderr.isTTY);
71
- }
72
-
73
- function formatBytes(bytes) {
74
- if (!bytes) return '';
75
- if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`;
76
- return `${Math.round(bytes / (1024 * 1024))} MB`;
77
- }
78
-
79
- async function promptUser(question) {
80
- return new Promise((resolve) => {
81
- const rl = readline.createInterface({ input: process.stdin, output: process.stderr });
82
- rl.question(question, (answer) => {
83
- rl.close();
84
- resolve(answer.trim().toLowerCase());
85
- });
86
- });
87
- }
88
-
89
- /**
90
- * Attempt to install a resource lazily, respecting cached consent.
91
- *
92
- * @param {string} id - resource id from the registry
93
- * @param {object} [opts]
94
- * @param {boolean} [opts.silent] - skip TTY prompt, use cached consent only
95
- * @returns {{ installed: boolean, fallback?: string, error?: string }}
96
- */
97
- export async function lazyInstall(id, { silent = false } = {}) {
98
- const resource = getResource(id);
99
- if (!resource) {
100
- return { installed: false, error: `unknown resource: ${id}` };
101
- }
102
-
103
- const probe = await probeResource(resource);
104
- if (probe.present && probe.healthy) {
105
- return { installed: true };
106
- }
107
-
108
- const { consentKey, displayName, downloadSize, fallback } = resource;
109
- const fallbackMsg = typeof fallback === 'function' ? fallback() : (fallback || '');
110
-
111
- const cached = readConsentFromEnv(consentKey);
112
-
113
- if (cached === 'never') {
114
- return { installed: false, fallback: fallbackMsg };
115
- }
116
-
117
- if (cached === 'yes') {
118
- return await performInstall(resource, fallbackMsg);
119
- }
120
-
121
- if (silent || !hasTty()) {
122
- return { installed: false, fallback: fallbackMsg };
123
- }
124
-
125
- const size = downloadSize ? ` (~${formatBytes(downloadSize)})` : '';
126
- const answer = await promptUser(
127
- `\n[construct] ${displayName} required${size}. Install now? [y/n/never] `
128
- );
129
-
130
- if (answer === 'y' || answer === 'yes') {
131
- writeConsentToEnv(consentKey, 'yes');
132
- return await performInstall(resource, fallbackMsg);
133
- }
134
-
135
- if (answer === 'never') {
136
- writeConsentToEnv(consentKey, 'never');
137
- process.stderr.write(
138
- `[construct] "${displayName}" will not be prompted again. Using fallback: ${fallbackMsg}\n`
139
- );
140
- return { installed: false, fallback: fallbackMsg };
141
- }
142
-
143
- return { installed: false, fallback: fallbackMsg };
144
- }
145
-
146
- async function performInstall(resource, fallbackMsg) {
147
- if (typeof resource.install !== 'function') {
148
- return { installed: false, fallback: fallbackMsg, error: 'resource has no install()' };
149
- }
150
- try {
151
- process.stderr.write(`[construct] Installing ${resource.displayName}...\n`);
152
- const result = await resource.install();
153
- if (result?.success) {
154
- process.stderr.write(`[construct] ${resource.displayName} installed.\n`);
155
- return { installed: true };
156
- }
157
- return { installed: false, fallback: fallbackMsg, error: result?.log };
158
- } catch (err) {
159
- return { installed: false, fallback: fallbackMsg, error: err?.message || String(err) };
160
- }
161
- }
@@ -1,198 +0,0 @@
1
- /**
2
- * lib/embed/jobs/vector-sync.mjs — Automatic embedding pipeline for observations.
3
- *
4
- * Runs on a schedule (default: every 5 minutes) to embed new/modified observations
5
- * and store them in Postgres (if available) and/or the local vector index.
6
- *
7
- * Design:
8
- * - Scans .cx/observations/ for items not yet embedded
9
- * - Embeds each using the configured model (local ONNX, OpenAI, Ollama)
10
- * - Stores in Postgres construct_observations table (if DATABASE_URL set)
11
- * - Falls back to local vectors.json if Postgres unavailable
12
- * - Batched processing to avoid overwhelming the embedding API
13
- * - Deduplicated via content hash comparison
14
- */
15
- import { existsSync, readFileSync, writeFileSync, statSync, mkdirSync } from 'node:fs';
16
- import { join, dirname } from 'node:path';
17
- import { homedir } from 'node:os';
18
-
19
- const DEFAULT_BATCH_SIZE = 32;
20
- const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
21
-
22
- /**
23
- * Find observations that haven't been embedded yet.
24
- * @param {string} rootDir
25
- * @returns {Array<{ id: string, project: string, role: string, category: string, summary: string, content: string, tags: string[], confidence: number, source: string, filePath: string }>}
26
- */
27
- function findUnembeddedObservations(rootDir) {
28
- const obsDir = join(rootDir, '.cx', 'observations');
29
- if (!existsSync(obsDir)) return [];
30
-
31
- const indexPath = join(obsDir, 'index.json');
32
- if (!existsSync(indexPath)) return [];
33
-
34
- let index;
35
- try {
36
- index = JSON.parse(readFileSync(indexPath, 'utf8'));
37
- } catch (err) {
38
- process.stderr.write('[jobs/vector-sync.mjs] findUnembedded: ' + (err?.message ?? String(err)) + '\n');
39
- return [];
40
- }
41
-
42
- // Check which observations have been embedded
43
- const vectorsPath = join(obsDir, 'vectors.json');
44
- const embeddedIds = new Set();
45
- if (existsSync(vectorsPath)) {
46
- try {
47
- const vectors = JSON.parse(readFileSync(vectorsPath, 'utf8'));
48
- for (const v of (vectors.records || vectors)) {
49
- if (v.id) embeddedIds.add(v.id);
50
- }
51
- } catch { /* ignore */ }
52
- }
53
-
54
- // Also check Postgres if available
55
- // (This is handled by the caller via VectorClient.isHealthy())
56
-
57
- return (index || [])
58
- .filter((obs) => obs.id && !embeddedIds.has(obs.id))
59
- .map((obs) => ({
60
- ...obs,
61
- filePath: join(obsDir, `${obs.id}.json`),
62
- }))
63
- .filter((obs) => existsSync(obs.filePath));
64
- }
65
-
66
- /**
67
- * Run the vector sync job.
68
- * @param {object} opts
69
- * @param {string} opts.rootDir - Root directory for observation store
70
- * @param {object} opts.env - Environment variables
71
- * @param {object} opts.logger - Optional logger with debug/info/error methods
72
- * @param {object} opts.vectorClient - Optional VectorClient instance for SQL storage
73
- * @returns {Promise<{ synced: number, errors: number, durationMs: number }>}
74
- */
75
- export async function runVectorSync({ rootDir, env = process.env, logger, vectorClient } = {}) {
76
- const startTime = Date.now();
77
- const batchSize = parseInt(env.CONSTRUCT_EMBEDDING_BATCH_SIZE, 10) || DEFAULT_BATCH_SIZE;
78
-
79
- try {
80
- const unembedded = findUnembeddedObservations(rootDir);
81
- if (!unembedded.length) {
82
- logger?.debug?.('Vector sync: no new observations to embed');
83
- return { synced: 0, errors: 0, durationMs: Date.now() - startTime };
84
- }
85
-
86
- let synced = 0;
87
- let errors = 0;
88
-
89
- // Dynamically import embedding engine
90
- const { embedBatch } = await import('../../storage/embeddings-engine.mjs');
91
-
92
- for (let i = 0; i < unembedded.length; i += batchSize) {
93
- const batch = unembedded.slice(i, i + batchSize);
94
- const texts = batch.map((o) => `${o.summary}\n${o.content}`);
95
-
96
- try {
97
- const embeddings = await embedBatch(texts, { env });
98
-
99
- for (let j = 0; j < batch.length; j++) {
100
- const obs = batch[j];
101
- const embedding = embeddings[j].embedding;
102
-
103
- // Store in Postgres if available
104
- if (vectorClient && await vectorClient.isHealthy()) {
105
- try {
106
- await vectorClient.storeObservation({
107
- id: obs.id,
108
- project: obs.project || rootDir.split('/').pop(),
109
- role: obs.role,
110
- category: obs.category,
111
- summary: obs.summary,
112
- content: obs.content,
113
- tags: obs.tags || [],
114
- confidence: obs.confidence ?? 0.8,
115
- source: obs.source,
116
- embedding,
117
- });
118
- } catch (err) {
119
- logger?.error?.(`Vector sync: failed to store observation ${obs.id} in SQL: ${err.message}`);
120
- errors++;
121
- continue;
122
- }
123
- }
124
-
125
- // Always update local vectors.json
126
- updateLocalVectorIndex(rootDir, obs, embedding);
127
- synced++;
128
- }
129
- } catch (err) {
130
- logger?.error?.(`Vector sync: batch embedding failed: ${err.message}`);
131
- errors += batch.length;
132
- }
133
- }
134
-
135
- const durationMs = Date.now() - startTime;
136
- logger?.info?.(`Vector sync: embedded ${synced} observation(s) in ${durationMs}ms (${errors} errors)`);
137
- return { synced, errors, durationMs };
138
- } catch (err) {
139
- const durationMs = Date.now() - startTime;
140
- logger?.error?.(`Vector sync failed: ${err.message}`);
141
- return { synced: 0, errors: 1, durationMs };
142
- }
143
- }
144
-
145
- /**
146
- * Update the local vectors.json index with a new embedding.
147
- * @param {string} rootDir
148
- * @param {object} obs - Observation record
149
- * @param {Float32Array} embedding
150
- */
151
- function updateLocalVectorIndex(rootDir, obs, embedding) {
152
- const obsDir = join(rootDir, '.cx', 'observations');
153
- const vectorsPath = join(obsDir, 'vectors.json');
154
-
155
- let vectors;
156
- if (existsSync(vectorsPath)) {
157
- try {
158
- vectors = JSON.parse(readFileSync(vectorsPath, 'utf8'));
159
- } catch (err) {
160
- process.stderr.write('[jobs/vector-sync.mjs] updateLocalVectorIndex: ' + (err?.message ?? String(err)) + '\n');
161
- vectors = { version: 1, model: 'hashing-bow-v1', updatedAt: null, records: [] };
162
- }
163
- } else {
164
- vectors = { version: 1, model: 'hashing-bow-v1', updatedAt: null, records: [] };
165
- }
166
-
167
- // Add or update the record
168
- const existingIdx = vectors.records.findIndex((r) => r.id === obs.id);
169
- const record = {
170
- id: obs.id,
171
- embedding: Array.from(embedding),
172
- summary: obs.summary,
173
- content: obs.content,
174
- tags: obs.tags || [],
175
- project: obs.project,
176
- role: obs.role,
177
- category: obs.category,
178
- updatedAt: new Date().toISOString(),
179
- };
180
-
181
- if (existingIdx >= 0) {
182
- vectors.records[existingIdx] = record;
183
- } else {
184
- vectors.records.push(record);
185
- }
186
-
187
- // Cap at 1000 records (sliding window)
188
- if (vectors.records.length > 1000) {
189
- vectors.records = vectors.records.slice(-1000);
190
- }
191
-
192
- vectors.updatedAt = new Date().toISOString();
193
-
194
- mkdirSync(dirname(vectorsPath), { recursive: true });
195
- writeFileSync(vectorsPath, JSON.stringify(vectors, null, 2), 'utf8');
196
- }
197
-
198
- export { DEFAULT_BATCH_SIZE, DEFAULT_INTERVAL_MS };