@geraldmaron/construct 1.0.17 → 1.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/bin/construct +26 -3
- package/commands/design/flow.md +2 -0
- package/commands/design/ui.md +2 -0
- package/db/schema/003_observation_reconciliation.sql +14 -0
- package/lib/bootstrap/resources.mjs +0 -1
- package/lib/cli-commands.mjs +33 -5
- package/lib/comment-lint.mjs +44 -0
- package/lib/contracts/validate.mjs +106 -0
- package/lib/decisions/enforced-baseline.json +23 -0
- package/lib/decisions/golden.mjs +87 -0
- package/lib/decisions/precedence.mjs +46 -0
- package/lib/decisions/registry.mjs +469 -0
- package/lib/deployment/parity-contract.mjs +148 -0
- package/lib/embed/cli.mjs +11 -0
- package/lib/embed/conflict-detection.mjs +4 -4
- package/lib/embed/customer-profiles.mjs +1 -1
- package/lib/embed/reconcile.mjs +60 -0
- package/lib/embed/semantic.mjs +5 -3
- package/lib/gates-audit.mjs +2 -2
- package/lib/hooks/config-protection.mjs +22 -3
- package/lib/hooks/guard-bash.mjs +1 -1
- package/lib/init-docs.mjs +1 -0
- package/lib/mode-commands.mjs +6 -8
- package/lib/observation-store.mjs +16 -2
- package/lib/opencode-telemetry.mjs +1 -1
- package/lib/roles/cli.mjs +10 -2
- package/lib/roles/gateway.mjs +50 -1
- package/lib/scheduler/index.mjs +31 -0
- package/lib/server/index.mjs +13 -3
- package/lib/server/static/index.html +1 -1
- package/lib/setup.mjs +6 -0
- package/lib/storage/hybrid-query.mjs +49 -38
- package/lib/storage/rrf.mjs +42 -0
- package/lib/storage/vector-client.mjs +18 -3
- package/lib/telemetry/backends/local.mjs +1 -1
- package/lib/telemetry/skill-calls.mjs +1 -1
- package/lib/templates/visual-requirements.mjs +84 -0
- package/package.json +12 -5
- package/rules/common/comments.md +3 -0
- package/rules/common/no-fabrication.md +3 -0
- package/rules/common/precedence.md +19 -0
- package/rules/common/research-sources.md +32 -0
- package/rules/common/research.md +59 -2
- package/skills/roles/data-engineer.pipeline.md +13 -1
- package/skills/roles/debugger.md +9 -0
- package/skills/roles/designer.accessibility.md +13 -3
- package/skills/roles/designer.md +10 -0
- package/skills/roles/engineer.platform.md +8 -0
- package/skills/roles/operator.md +10 -1
- package/skills/roles/operator.release.md +8 -0
- package/skills/roles/operator.sre.md +10 -1
- package/skills/roles/orchestrator.md +9 -2
- package/skills/roles/product-manager.business-strategy.md +10 -1
- package/skills/roles/researcher.explorer.md +12 -1
- package/skills/roles/researcher.ux.md +13 -1
- package/skills/roles/reviewer.devil-advocate.md +14 -2
- package/skills/roles/reviewer.evaluator.md +17 -4
- package/skills/roles/reviewer.trace.md +12 -1
- package/skills/roles/security.legal-compliance.md +8 -0
- package/skills/roles/security.md +11 -0
- package/specialists/contracts.json +18 -0
- package/specialists/prompts/cx-architect.md +1 -0
- package/specialists/prompts/cx-designer.md +1 -1
- package/specialists/prompts/cx-operations.md +1 -0
- package/specialists/prompts/cx-rd-lead.md +1 -0
- package/specialists/prompts/cx-researcher.md +4 -2
- package/specialists/prompts/cx-sre.md +1 -0
- package/specialists/registry.json +2 -2
- package/templates/docs/backlog-proposal.md +1 -1
- package/templates/docs/customer-profile.md +1 -1
- package/templates/docs/evidence-brief.md +5 -1
- package/templates/docs/incident-report.md +37 -21
- package/templates/docs/prfaq.md +2 -2
- package/templates/docs/product-intelligence-report.md +1 -1
- package/templates/docs/research-brief.md +8 -6
- package/templates/docs/research-finding.md +32 -7
- package/templates/docs/rfc.md +13 -1
- package/templates/docs/runbook.md +20 -1
- package/templates/docs/signal-brief.md +4 -1
- package/templates/docs/skill-artifact.md +27 -7
- package/templates/docs/strategy.md +23 -2
- package/templates/docs/test-plan.md +96 -0
- package/lib/bootstrap/lazy-install.mjs +0 -161
- package/lib/embed/jobs/vector-sync.mjs +0 -198
- package/lib/knowledge/postgres-search.mjs +0 -132
- package/lib/services/pattern-promotion-service.mjs +0 -167
- package/lib/storage/unified-storage.mjs +0 -550
|
@@ -1,26 +1,51 @@
|
|
|
1
1
|
---
|
|
2
2
|
kind: research-finding
|
|
3
3
|
topic: "<short topic line>"
|
|
4
|
-
confidence:
|
|
4
|
+
confidence: high | medium | low
|
|
5
5
|
sources: []
|
|
6
6
|
created: <ISO timestamp set by construct knowledge add>
|
|
7
|
-
expiresAt: <ISO timestamp, default created + 90d>
|
|
7
|
+
expiresAt: <ISO timestamp, default created + 90d; shorten for fast-moving topics>
|
|
8
8
|
profile: <active profile id>
|
|
9
9
|
---
|
|
10
10
|
|
|
11
|
+
<!--
|
|
12
|
+
A lightweight but rigorous knowledge-store entry. Same evidence standard as a
|
|
13
|
+
research brief, condensed: observation separated from inference, every finding
|
|
14
|
+
cited, confidence tied to source grade, an explicit refresh trigger. See
|
|
15
|
+
rules/common/research.md.
|
|
16
|
+
-->
|
|
17
|
+
|
|
18
|
+
## SOURCES
|
|
19
|
+
|
|
20
|
+
| Title / Path | Class | Reliability | Credibility | Date | URL | Verified |
|
|
21
|
+
|---|---|---|---|---|---|---|
|
|
22
|
+
| {source} | primary / secondary / tertiary | A–F | 1–6 | {YYYY-MM-DD} | {url} | yes / no |
|
|
23
|
+
|
|
11
24
|
## FINDINGS
|
|
12
25
|
|
|
13
|
-
|
|
14
|
-
|
|
26
|
+
<!-- Observation only: what the source states. Each finding cites a source row above. -->
|
|
27
|
+
|
|
28
|
+
- <Finding 1 [source: …]>
|
|
29
|
+
- <Finding 2 [source: …]>
|
|
15
30
|
|
|
16
31
|
## INFERENCES
|
|
17
32
|
|
|
18
|
-
|
|
33
|
+
<!-- What is concluded beyond what any single source says. Labeled as inference, not fact. -->
|
|
34
|
+
|
|
35
|
+
- <Inference, with the findings it rests on>
|
|
36
|
+
|
|
37
|
+
## CONFIDENCE
|
|
38
|
+
|
|
39
|
+
<!-- high / medium / low, with the reasoning: tie it to the Admiralty grade of the sources (research.md §10). high only on A1/A2/B1. Name the strongest counter-evidence. -->
|
|
19
40
|
|
|
20
41
|
## GAPS
|
|
21
42
|
|
|
22
|
-
- <What
|
|
43
|
+
- <What could not be confirmed and would change the conclusion if known>
|
|
23
44
|
|
|
24
45
|
## RECOMMENDATION
|
|
25
46
|
|
|
26
|
-
- <Next action based on the evidence available>
|
|
47
|
+
- <Next action based on the evidence available, and the threshold that would change it>
|
|
48
|
+
|
|
49
|
+
## REFRESH
|
|
50
|
+
|
|
51
|
+
<!-- When this finding should be re-verified: the expiresAt date, or the event that would invalidate it (a new release, a superseding paper). -->
|
package/templates/docs/rfc.md
CHANGED
|
@@ -22,7 +22,19 @@ contract, API, schema, or protocol consumed by other systems.
|
|
|
22
22
|
<!-- What problem or limitation drives this proposal? Include evidence: incidents, performance data, support tickets, developer friction. Explain why the status quo is not acceptable. -->
|
|
23
23
|
|
|
24
24
|
## Proposed design
|
|
25
|
-
<!-- The concrete proposal. Enough detail for reviewers to evaluate feasibility and tradeoffs.
|
|
25
|
+
<!-- The concrete proposal. Enough detail for reviewers to evaluate feasibility and tradeoffs. The sequence below shows the proposed behavior in context — include the happy path and at least one error path. -->
|
|
26
|
+
|
|
27
|
+
```mermaid
|
|
28
|
+
sequenceDiagram
|
|
29
|
+
participant Client
|
|
30
|
+
participant Service
|
|
31
|
+
participant Store
|
|
32
|
+
Client->>Service: request
|
|
33
|
+
Service->>Store: read or write
|
|
34
|
+
Store-->>Service: result
|
|
35
|
+
Service-->>Client: response
|
|
36
|
+
Note over Service,Store: error path — Store unavailable → Service returns degraded response
|
|
37
|
+
```
|
|
26
38
|
|
|
27
39
|
## Tradeoffs and alternatives
|
|
28
40
|
<!-- The other credible designs considered. For each: what it is, why it was not chosen, and under what conditions it would be preferred. No strawmen. -->
|
|
@@ -14,8 +14,27 @@
|
|
|
14
14
|
## Impact
|
|
15
15
|
<!-- Who is affected and how badly. Data loss? Degraded performance? Complete outage? -->
|
|
16
16
|
|
|
17
|
+
## Severity and response
|
|
18
|
+
<!-- Map each severity to the response it triggers: page urgency, comms cadence, and error-budget consequence. A SEV-1 pages immediately and freezes related releases per the error-budget policy (https://sre.google/workbook/error-budget-policy/); a SEV-3 is handled in business hours. -->
|
|
19
|
+
|
|
20
|
+
| Severity | Trigger condition | Page within | Comms | Error budget |
|
|
21
|
+
|----------|-------------------|-------------|-------|--------------|
|
|
22
|
+
| SEV-1 | {full outage / data loss / SLO breach} | 5 min | exec + status page | breach → freeze releases |
|
|
23
|
+
| SEV-2 | {major degradation, workaround exists} | 15 min | team + stakeholders | partial spend |
|
|
24
|
+
| SEV-3 | {minor / single-tenant / cosmetic} | business hours | team channel | none |
|
|
25
|
+
|
|
17
26
|
## Diagnostic steps
|
|
18
|
-
<!-- Ordered checks from cheapest to most expensive. Each step: what to check, how to check it, what the answer means. -->
|
|
27
|
+
<!-- Ordered checks from cheapest to most expensive. Each step: what to check, how to check it, what the answer means. Keep the decision tree below in sync with the steps. -->
|
|
28
|
+
|
|
29
|
+
```mermaid
|
|
30
|
+
flowchart TD
|
|
31
|
+
A[Alert fires] --> B{Error rate elevated?}
|
|
32
|
+
B -->|Yes| C[Check upstream dependency health]
|
|
33
|
+
B -->|No| D{User-visible impact?}
|
|
34
|
+
C --> E[Apply remediation]
|
|
35
|
+
D -->|Yes| C
|
|
36
|
+
D -->|No| F[Monitor and stand down]
|
|
37
|
+
```
|
|
19
38
|
|
|
20
39
|
## Remediation
|
|
21
40
|
<!-- The fix. Step-by-step, with exact commands or UI paths. Include expected output for each step. -->
|
|
@@ -13,7 +13,10 @@ without pretending the requirements are ready.
|
|
|
13
13
|
<!-- One or two paragraphs. What appears to be happening, and where the signal came from. -->
|
|
14
14
|
|
|
15
15
|
## Evidence
|
|
16
|
-
<!-- Cite the concrete notes, tickets, customer quotes, traces, or analytics
|
|
16
|
+
<!-- Cite the concrete notes, tickets, customer quotes, traces, or analytics. Grade each source with its Admiralty reliability/credibility (rules/common/research.md §10); for community sources note corroboration and engagement. The signal strength above should follow from the grades, not precede them. -->
|
|
17
|
+
|
|
18
|
+
## Counter-signal
|
|
19
|
+
<!-- The strongest evidence against the signal, or its absence. A signal no one tried to disconfirm is weaker than it looks. -->
|
|
17
20
|
|
|
18
21
|
## Why it matters
|
|
19
22
|
<!-- What product, customer, or business decision this could affect if the signal strengthens. -->
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: <verb-led-id>
|
|
3
3
|
scope: "<one paragraph: what it covers, what it does not>"
|
|
4
|
-
observable_outcome: <how someone outside the role tells this skill happened>
|
|
4
|
+
observable_outcome: <how someone outside the role tells this skill happened — a concrete artifact, not "understands X">
|
|
5
|
+
prerequisites:
|
|
6
|
+
- <skill-id or capability that must exist before this one is useful>
|
|
5
7
|
provenance:
|
|
6
|
-
- <citation: post-mortem, public framework, competency model>
|
|
8
|
+
- <citation: post-mortem, public framework, competency model, with URL>
|
|
7
9
|
roles:
|
|
8
10
|
- <role-id that uses this skill>
|
|
9
11
|
---
|
|
@@ -12,16 +14,34 @@ roles:
|
|
|
12
14
|
|
|
13
15
|
## What this skill produces
|
|
14
16
|
|
|
15
|
-
<Concrete output
|
|
17
|
+
<Concrete output, Bloom-style: not "knows X" but "produces Y". Name the artifact and what makes it correct. Example: "a runbook whose diagnostic flowchart branches to a fix in ≤3 steps", not "understands incident response".>
|
|
16
18
|
|
|
17
19
|
## When to invoke it
|
|
18
20
|
|
|
19
|
-
<Triggers. What the operator or
|
|
21
|
+
<Triggers. What the operator or upstream persona is doing when this skill is the right call. Include the signal that distinguishes it from adjacent skills.>
|
|
20
22
|
|
|
21
|
-
##
|
|
23
|
+
## Prerequisites and composition
|
|
22
24
|
|
|
23
|
-
<
|
|
25
|
+
<What must be true or done first, and the skills typically chained before and after. A skill with unmet prerequisites produces confident-looking but wrong output.>
|
|
26
|
+
|
|
27
|
+
## Competency rubric
|
|
28
|
+
|
|
29
|
+
<What the skill looks like at each level. The author of an artifact should be able to place their work on this scale.>
|
|
30
|
+
|
|
31
|
+
| Level | What it looks like |
|
|
32
|
+
|-------|--------------------|
|
|
33
|
+
| Novice | <produces the artifact but misses edge cases / evidence / failure paths> |
|
|
34
|
+
| Competent | <covers the happy path and the common failure modes; cites sources> |
|
|
35
|
+
| Expert | <anticipates the non-obvious failure, calibrates confidence, leaves the next person a reproducible trail> |
|
|
36
|
+
|
|
37
|
+
## Failure modes
|
|
38
|
+
|
|
39
|
+
<How this skill breaks when misapplied — distinct from anti-patterns. What does a wrong-but-plausible output look like, and what signal reveals it? When should the operator escalate or stop?>
|
|
24
40
|
|
|
25
41
|
## Anti-patterns
|
|
26
42
|
|
|
27
|
-
<Ways this skill gets misapplied; what looks similar but is not this.>
|
|
43
|
+
<Ways this skill gets misapplied; what looks similar but is not this skill.>
|
|
44
|
+
|
|
45
|
+
## Worked example
|
|
46
|
+
|
|
47
|
+
<A short, concrete instance: the trigger, the artifact produced, and why it meets the observable outcome. One real example removes more ambiguity than a page of guidance.>
|
|
@@ -15,8 +15,8 @@ Owning workflow: skills/docs/strategy-workflow.md.
|
|
|
15
15
|
<!-- 1–3 sentences. Where are we going in 3–5 years? -->
|
|
16
16
|
|
|
17
17
|
## Bets
|
|
18
|
-
<!-- Explicit choices. Each bet: what we're doing and
|
|
19
|
-
- **Bet 1**: …
|
|
18
|
+
<!-- Explicit choices. Each bet: what we're doing, why, the leading indicator that tells us early if it's working, and the kill criterion that ends it. A bet with no kill criterion is a belief, not a bet. -->
|
|
19
|
+
- **Bet 1**: … **Why**: … **Leading indicator**: … **Kill criterion**: {what observation would make us stop}
|
|
20
20
|
- **Bet 2**: …
|
|
21
21
|
|
|
22
22
|
## Non-bets
|
|
@@ -29,6 +29,27 @@ Owning workflow: skills/docs/strategy-workflow.md.
|
|
|
29
29
|
## North Star Metric
|
|
30
30
|
<!-- Single measurable outcome. Format: Metric / Baseline / Target / Owner -->
|
|
31
31
|
|
|
32
|
+
## Metrics
|
|
33
|
+
<!-- The North Star is lagging. Name the leading indicators that move first and predict it, so a failing bet is visible before the lagging metric confirms it. -->
|
|
34
|
+
|
|
35
|
+
| Metric | Type (leading/lagging) | Baseline | Target | Owner |
|
|
36
|
+
|--------|------------------------|----------|--------|-------|
|
|
37
|
+
| {name} | leading / lagging | {value} | {value} | {who} |
|
|
38
|
+
|
|
39
|
+
## Milestones
|
|
40
|
+
<!-- What must be true by when. Each milestone is a checkpoint that confirms or kills a bet. -->
|
|
41
|
+
|
|
42
|
+
| Milestone | By when | Confirms / kills |
|
|
43
|
+
|-----------|---------|------------------|
|
|
44
|
+
| {what} | {date} | {which bet} |
|
|
45
|
+
|
|
46
|
+
## Risks
|
|
47
|
+
<!-- What could invalidate the strategy. Risk register, not a worry list. -->
|
|
48
|
+
|
|
49
|
+
| Risk | Likelihood | Impact | Mitigation / leading signal |
|
|
50
|
+
|------|-----------|--------|------------------------------|
|
|
51
|
+
| {what} | high/med/low | high/med/low | {response} |
|
|
52
|
+
|
|
32
53
|
## Competitive Positioning
|
|
33
54
|
<!-- How we differentiate. 2–4 sentences. -->
|
|
34
55
|
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Test Plan: {feature or system name}
|
|
2
|
+
|
|
3
|
+
- **Date**: {YYYY-MM-DD}
|
|
4
|
+
- **Owner**: {name or role}
|
|
5
|
+
- **Status**: draft | in-review | approved | executed
|
|
6
|
+
- **Related**: {PRD / ADR paths or none}
|
|
7
|
+
|
|
8
|
+
<!--
|
|
9
|
+
Use this when a feature, migration, or system change needs a deliberate
|
|
10
|
+
verification strategy before it ships. The plan is the contract between
|
|
11
|
+
what was specified and what will be proven; every scenario below must
|
|
12
|
+
trace to a requirement or an observed risk, never to guesswork.
|
|
13
|
+
|
|
14
|
+
Before drafting, read rules/common/framing.md and rules/common/no-fabrication.md.
|
|
15
|
+
Acceptance criteria must be binary pass/fail. If a criterion cannot be
|
|
16
|
+
checked without asking the author, it is not done. Cite the requirement
|
|
17
|
+
(FR/NFR id, ADR decision, ticket) each scenario verifies.
|
|
18
|
+
-->
|
|
19
|
+
|
|
20
|
+
## Scope
|
|
21
|
+
<!--
|
|
22
|
+
What is under test and what is explicitly not. Name the components, surfaces,
|
|
23
|
+
or behaviors covered. The out-of-scope list protects reviewer attention and
|
|
24
|
+
makes coverage gaps deliberate rather than accidental.
|
|
25
|
+
-->
|
|
26
|
+
|
|
27
|
+
| | Description |
|
|
28
|
+
|---|---|
|
|
29
|
+
| **In scope** | <what this plan verifies> |
|
|
30
|
+
| **Out of scope** | <related behavior deliberately not tested here, and why> |
|
|
31
|
+
|
|
32
|
+
## Test strategy
|
|
33
|
+
<!--
|
|
34
|
+
The levels of testing applied and why each is justified. Map levels to risk:
|
|
35
|
+
unit for logic, integration for contracts and boundaries, end-to-end for
|
|
36
|
+
user-visible flows, manual/exploratory for what automation cannot reach.
|
|
37
|
+
State what each level is responsible for so coverage gaps are visible.
|
|
38
|
+
-->
|
|
39
|
+
|
|
40
|
+
| Level | What it covers | Why it lives at this level |
|
|
41
|
+
|---|---|---|
|
|
42
|
+
| Unit | <logic, pure functions, edge cases> | <reason> |
|
|
43
|
+
| Integration | <contracts, boundaries, data access> | <reason> |
|
|
44
|
+
| End-to-end | <user-visible flows> | <reason> |
|
|
45
|
+
| Manual / exploratory | <what automation cannot reach> | <reason> |
|
|
46
|
+
|
|
47
|
+
## Key scenarios
|
|
48
|
+
<!--
|
|
49
|
+
The scenarios that prove the change works. Each scenario names the requirement
|
|
50
|
+
it verifies, the setup, the action, and the binary acceptance criterion. A
|
|
51
|
+
reviewer must be able to run or read the scenario and decide pass/fail without
|
|
52
|
+
asking the author.
|
|
53
|
+
-->
|
|
54
|
+
|
|
55
|
+
| ID | Scenario | Verifies | Acceptance (binary pass/fail) |
|
|
56
|
+
|---|---|---|---|
|
|
57
|
+
| TC-1 | <setup, action, expected result> | <FR/NFR id or risk> | <observable condition> |
|
|
58
|
+
| TC-2 | <...> | <...> | <...> |
|
|
59
|
+
|
|
60
|
+
## Risks and edge cases
|
|
61
|
+
<!--
|
|
62
|
+
The failure modes and boundary conditions worth explicit attention: empty
|
|
63
|
+
inputs, concurrency, partial failure, large data, permission boundaries,
|
|
64
|
+
rollback. For each, state how the plan covers it or why the residual risk
|
|
65
|
+
is accepted.
|
|
66
|
+
-->
|
|
67
|
+
|
|
68
|
+
| Risk / edge case | Likelihood | Impact | Coverage or accepted reason |
|
|
69
|
+
|---|---|---|---|
|
|
70
|
+
| <condition> | low / med / high | low / med / high | <how it is tested, or why accepted> |
|
|
71
|
+
|
|
72
|
+
## Environments and data
|
|
73
|
+
<!--
|
|
74
|
+
Where tests run and what data they use. Name the environments, fixtures,
|
|
75
|
+
seed data, and any external dependencies (sandboxes, stubs, recorded
|
|
76
|
+
responses). Flag data that must be synthetic or anonymized.
|
|
77
|
+
-->
|
|
78
|
+
|
|
79
|
+
## Entry and exit criteria
|
|
80
|
+
<!--
|
|
81
|
+
Entry: what must be true before testing starts (build available, environment
|
|
82
|
+
provisioned, dependencies stubbed). Exit: what must be true to call testing
|
|
83
|
+
done (all blocking scenarios pass, known issues triaged, coverage threshold
|
|
84
|
+
met). Both are checklists a reviewer can verify.
|
|
85
|
+
-->
|
|
86
|
+
|
|
87
|
+
**Entry criteria**
|
|
88
|
+
|
|
89
|
+
- <condition that must hold before execution begins>
|
|
90
|
+
|
|
91
|
+
**Exit criteria**
|
|
92
|
+
|
|
93
|
+
- <condition that must hold to declare the plan executed>
|
|
94
|
+
|
|
95
|
+
## References
|
|
96
|
+
<!-- PRD, ADR, RFC, related test plans, prior incidents. -->
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* lib/bootstrap/lazy-install.mjs — on-demand resource installer with consent gating.
|
|
3
|
-
*
|
|
4
|
-
* When a runtime code path needs a resource that is not present, call
|
|
5
|
-
* `lazyInstall(id)`. The function:
|
|
6
|
-
*
|
|
7
|
-
* 1. Reads cached operator consent from config.env (consent key set by the
|
|
8
|
-
* resource definition in `lib/bootstrap/resources.mjs`).
|
|
9
|
-
* 2. If consent is 'never', returns { installed: false, fallback: <text> }
|
|
10
|
-
* without prompting or printing anything.
|
|
11
|
-
* 3. If a TTY is available, prompts the user with the resource displayName,
|
|
12
|
-
* estimated download size, and [y/n/never].
|
|
13
|
-
* 4. On 'y' or an existing 'yes' consent: calls resource.install(), waits
|
|
14
|
-
* for it to complete, updates config.env, and returns { installed: true }.
|
|
15
|
-
* 5. On 'n': returns { installed: false, fallback: <text> } without saving.
|
|
16
|
-
* 6. On 'never': writes 'never' to config.env and returns { installed: false }.
|
|
17
|
-
*
|
|
18
|
-
* Hooks that run without a TTY (most hook invocations) skip to step 2 and
|
|
19
|
-
* use the fallback silently.
|
|
20
|
-
*
|
|
21
|
-
* Consent is cached per-resource in config.env so the prompt never repeats
|
|
22
|
-
* across process restarts once the user has answered.
|
|
23
|
-
*/
|
|
24
|
-
|
|
25
|
-
import fs from 'node:fs';
|
|
26
|
-
import path from 'node:path';
|
|
27
|
-
import os from 'node:os';
|
|
28
|
-
import readline from 'node:readline';
|
|
29
|
-
|
|
30
|
-
import { getResource, probeResource } from './resources.mjs';
|
|
31
|
-
|
|
32
|
-
const CONFIG_ENV = path.join(os.homedir(), '.construct', 'config.env');
|
|
33
|
-
|
|
34
|
-
function readConfigEnv() {
|
|
35
|
-
try {
|
|
36
|
-
return fs.readFileSync(CONFIG_ENV, 'utf8');
|
|
37
|
-
} catch {
|
|
38
|
-
return '';
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
function readConsentFromEnv(consentKey) {
|
|
43
|
-
const lines = readConfigEnv().split('\n');
|
|
44
|
-
for (const line of lines) {
|
|
45
|
-
const trimmed = line.trim();
|
|
46
|
-
if (!trimmed || trimmed.startsWith('#')) continue;
|
|
47
|
-
const eq = trimmed.indexOf('=');
|
|
48
|
-
if (eq === -1) continue;
|
|
49
|
-
const k = trimmed.slice(0, eq).trim();
|
|
50
|
-
const v = trimmed.slice(eq + 1).trim().replace(/^['"]|['"]$/g, '');
|
|
51
|
-
if (k === consentKey) return v;
|
|
52
|
-
}
|
|
53
|
-
return null;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
function writeConsentToEnv(consentKey, value) {
|
|
57
|
-
let content = readConfigEnv();
|
|
58
|
-
const regex = new RegExp(`^${consentKey}=.*$`, 'm');
|
|
59
|
-
const line = `${consentKey}=${value}`;
|
|
60
|
-
if (regex.test(content)) {
|
|
61
|
-
content = content.replace(regex, line);
|
|
62
|
-
} else {
|
|
63
|
-
content = content.trimEnd() + (content ? '\n' : '') + line + '\n';
|
|
64
|
-
}
|
|
65
|
-
fs.mkdirSync(path.dirname(CONFIG_ENV), { recursive: true });
|
|
66
|
-
fs.writeFileSync(CONFIG_ENV, content, { mode: 0o600 });
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function hasTty() {
|
|
70
|
-
return Boolean(process.stdin.isTTY && process.stderr.isTTY);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
function formatBytes(bytes) {
|
|
74
|
-
if (!bytes) return '';
|
|
75
|
-
if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`;
|
|
76
|
-
return `${Math.round(bytes / (1024 * 1024))} MB`;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
async function promptUser(question) {
|
|
80
|
-
return new Promise((resolve) => {
|
|
81
|
-
const rl = readline.createInterface({ input: process.stdin, output: process.stderr });
|
|
82
|
-
rl.question(question, (answer) => {
|
|
83
|
-
rl.close();
|
|
84
|
-
resolve(answer.trim().toLowerCase());
|
|
85
|
-
});
|
|
86
|
-
});
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Attempt to install a resource lazily, respecting cached consent.
|
|
91
|
-
*
|
|
92
|
-
* @param {string} id - resource id from the registry
|
|
93
|
-
* @param {object} [opts]
|
|
94
|
-
* @param {boolean} [opts.silent] - skip TTY prompt, use cached consent only
|
|
95
|
-
* @returns {{ installed: boolean, fallback?: string, error?: string }}
|
|
96
|
-
*/
|
|
97
|
-
export async function lazyInstall(id, { silent = false } = {}) {
|
|
98
|
-
const resource = getResource(id);
|
|
99
|
-
if (!resource) {
|
|
100
|
-
return { installed: false, error: `unknown resource: ${id}` };
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
const probe = await probeResource(resource);
|
|
104
|
-
if (probe.present && probe.healthy) {
|
|
105
|
-
return { installed: true };
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
const { consentKey, displayName, downloadSize, fallback } = resource;
|
|
109
|
-
const fallbackMsg = typeof fallback === 'function' ? fallback() : (fallback || '');
|
|
110
|
-
|
|
111
|
-
const cached = readConsentFromEnv(consentKey);
|
|
112
|
-
|
|
113
|
-
if (cached === 'never') {
|
|
114
|
-
return { installed: false, fallback: fallbackMsg };
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
if (cached === 'yes') {
|
|
118
|
-
return await performInstall(resource, fallbackMsg);
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
if (silent || !hasTty()) {
|
|
122
|
-
return { installed: false, fallback: fallbackMsg };
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
const size = downloadSize ? ` (~${formatBytes(downloadSize)})` : '';
|
|
126
|
-
const answer = await promptUser(
|
|
127
|
-
`\n[construct] ${displayName} required${size}. Install now? [y/n/never] `
|
|
128
|
-
);
|
|
129
|
-
|
|
130
|
-
if (answer === 'y' || answer === 'yes') {
|
|
131
|
-
writeConsentToEnv(consentKey, 'yes');
|
|
132
|
-
return await performInstall(resource, fallbackMsg);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
if (answer === 'never') {
|
|
136
|
-
writeConsentToEnv(consentKey, 'never');
|
|
137
|
-
process.stderr.write(
|
|
138
|
-
`[construct] "${displayName}" will not be prompted again. Using fallback: ${fallbackMsg}\n`
|
|
139
|
-
);
|
|
140
|
-
return { installed: false, fallback: fallbackMsg };
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
return { installed: false, fallback: fallbackMsg };
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
async function performInstall(resource, fallbackMsg) {
|
|
147
|
-
if (typeof resource.install !== 'function') {
|
|
148
|
-
return { installed: false, fallback: fallbackMsg, error: 'resource has no install()' };
|
|
149
|
-
}
|
|
150
|
-
try {
|
|
151
|
-
process.stderr.write(`[construct] Installing ${resource.displayName}...\n`);
|
|
152
|
-
const result = await resource.install();
|
|
153
|
-
if (result?.success) {
|
|
154
|
-
process.stderr.write(`[construct] ${resource.displayName} installed.\n`);
|
|
155
|
-
return { installed: true };
|
|
156
|
-
}
|
|
157
|
-
return { installed: false, fallback: fallbackMsg, error: result?.log };
|
|
158
|
-
} catch (err) {
|
|
159
|
-
return { installed: false, fallback: fallbackMsg, error: err?.message || String(err) };
|
|
160
|
-
}
|
|
161
|
-
}
|
|
@@ -1,198 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* lib/embed/jobs/vector-sync.mjs — Automatic embedding pipeline for observations.
|
|
3
|
-
*
|
|
4
|
-
* Runs on a schedule (default: every 5 minutes) to embed new/modified observations
|
|
5
|
-
* and store them in Postgres (if available) and/or the local vector index.
|
|
6
|
-
*
|
|
7
|
-
* Design:
|
|
8
|
-
* - Scans .cx/observations/ for items not yet embedded
|
|
9
|
-
* - Embeds each using the configured model (local ONNX, OpenAI, Ollama)
|
|
10
|
-
* - Stores in Postgres construct_observations table (if DATABASE_URL set)
|
|
11
|
-
* - Falls back to local vectors.json if Postgres unavailable
|
|
12
|
-
* - Batched processing to avoid overwhelming the embedding API
|
|
13
|
-
* - Deduplicated via content hash comparison
|
|
14
|
-
*/
|
|
15
|
-
import { existsSync, readFileSync, writeFileSync, statSync, mkdirSync } from 'node:fs';
|
|
16
|
-
import { join, dirname } from 'node:path';
|
|
17
|
-
import { homedir } from 'node:os';
|
|
18
|
-
|
|
19
|
-
const DEFAULT_BATCH_SIZE = 32;
|
|
20
|
-
const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
|
|
21
|
-
|
|
22
|
-
/**
|
|
23
|
-
* Find observations that haven't been embedded yet.
|
|
24
|
-
* @param {string} rootDir
|
|
25
|
-
* @returns {Array<{ id: string, project: string, role: string, category: string, summary: string, content: string, tags: string[], confidence: number, source: string, filePath: string }>}
|
|
26
|
-
*/
|
|
27
|
-
function findUnembeddedObservations(rootDir) {
|
|
28
|
-
const obsDir = join(rootDir, '.cx', 'observations');
|
|
29
|
-
if (!existsSync(obsDir)) return [];
|
|
30
|
-
|
|
31
|
-
const indexPath = join(obsDir, 'index.json');
|
|
32
|
-
if (!existsSync(indexPath)) return [];
|
|
33
|
-
|
|
34
|
-
let index;
|
|
35
|
-
try {
|
|
36
|
-
index = JSON.parse(readFileSync(indexPath, 'utf8'));
|
|
37
|
-
} catch (err) {
|
|
38
|
-
process.stderr.write('[jobs/vector-sync.mjs] findUnembedded: ' + (err?.message ?? String(err)) + '\n');
|
|
39
|
-
return [];
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
// Check which observations have been embedded
|
|
43
|
-
const vectorsPath = join(obsDir, 'vectors.json');
|
|
44
|
-
const embeddedIds = new Set();
|
|
45
|
-
if (existsSync(vectorsPath)) {
|
|
46
|
-
try {
|
|
47
|
-
const vectors = JSON.parse(readFileSync(vectorsPath, 'utf8'));
|
|
48
|
-
for (const v of (vectors.records || vectors)) {
|
|
49
|
-
if (v.id) embeddedIds.add(v.id);
|
|
50
|
-
}
|
|
51
|
-
} catch { /* ignore */ }
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
// Also check Postgres if available
|
|
55
|
-
// (This is handled by the caller via VectorClient.isHealthy())
|
|
56
|
-
|
|
57
|
-
return (index || [])
|
|
58
|
-
.filter((obs) => obs.id && !embeddedIds.has(obs.id))
|
|
59
|
-
.map((obs) => ({
|
|
60
|
-
...obs,
|
|
61
|
-
filePath: join(obsDir, `${obs.id}.json`),
|
|
62
|
-
}))
|
|
63
|
-
.filter((obs) => existsSync(obs.filePath));
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* Run the vector sync job.
|
|
68
|
-
* @param {object} opts
|
|
69
|
-
* @param {string} opts.rootDir - Root directory for observation store
|
|
70
|
-
* @param {object} opts.env - Environment variables
|
|
71
|
-
* @param {object} opts.logger - Optional logger with debug/info/error methods
|
|
72
|
-
* @param {object} opts.vectorClient - Optional VectorClient instance for SQL storage
|
|
73
|
-
* @returns {Promise<{ synced: number, errors: number, durationMs: number }>}
|
|
74
|
-
*/
|
|
75
|
-
export async function runVectorSync({ rootDir, env = process.env, logger, vectorClient } = {}) {
|
|
76
|
-
const startTime = Date.now();
|
|
77
|
-
const batchSize = parseInt(env.CONSTRUCT_EMBEDDING_BATCH_SIZE, 10) || DEFAULT_BATCH_SIZE;
|
|
78
|
-
|
|
79
|
-
try {
|
|
80
|
-
const unembedded = findUnembeddedObservations(rootDir);
|
|
81
|
-
if (!unembedded.length) {
|
|
82
|
-
logger?.debug?.('Vector sync: no new observations to embed');
|
|
83
|
-
return { synced: 0, errors: 0, durationMs: Date.now() - startTime };
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
let synced = 0;
|
|
87
|
-
let errors = 0;
|
|
88
|
-
|
|
89
|
-
// Dynamically import embedding engine
|
|
90
|
-
const { embedBatch } = await import('../../storage/embeddings-engine.mjs');
|
|
91
|
-
|
|
92
|
-
for (let i = 0; i < unembedded.length; i += batchSize) {
|
|
93
|
-
const batch = unembedded.slice(i, i + batchSize);
|
|
94
|
-
const texts = batch.map((o) => `${o.summary}\n${o.content}`);
|
|
95
|
-
|
|
96
|
-
try {
|
|
97
|
-
const embeddings = await embedBatch(texts, { env });
|
|
98
|
-
|
|
99
|
-
for (let j = 0; j < batch.length; j++) {
|
|
100
|
-
const obs = batch[j];
|
|
101
|
-
const embedding = embeddings[j].embedding;
|
|
102
|
-
|
|
103
|
-
// Store in Postgres if available
|
|
104
|
-
if (vectorClient && await vectorClient.isHealthy()) {
|
|
105
|
-
try {
|
|
106
|
-
await vectorClient.storeObservation({
|
|
107
|
-
id: obs.id,
|
|
108
|
-
project: obs.project || rootDir.split('/').pop(),
|
|
109
|
-
role: obs.role,
|
|
110
|
-
category: obs.category,
|
|
111
|
-
summary: obs.summary,
|
|
112
|
-
content: obs.content,
|
|
113
|
-
tags: obs.tags || [],
|
|
114
|
-
confidence: obs.confidence ?? 0.8,
|
|
115
|
-
source: obs.source,
|
|
116
|
-
embedding,
|
|
117
|
-
});
|
|
118
|
-
} catch (err) {
|
|
119
|
-
logger?.error?.(`Vector sync: failed to store observation ${obs.id} in SQL: ${err.message}`);
|
|
120
|
-
errors++;
|
|
121
|
-
continue;
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
// Always update local vectors.json
|
|
126
|
-
updateLocalVectorIndex(rootDir, obs, embedding);
|
|
127
|
-
synced++;
|
|
128
|
-
}
|
|
129
|
-
} catch (err) {
|
|
130
|
-
logger?.error?.(`Vector sync: batch embedding failed: ${err.message}`);
|
|
131
|
-
errors += batch.length;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const durationMs = Date.now() - startTime;
|
|
136
|
-
logger?.info?.(`Vector sync: embedded ${synced} observation(s) in ${durationMs}ms (${errors} errors)`);
|
|
137
|
-
return { synced, errors, durationMs };
|
|
138
|
-
} catch (err) {
|
|
139
|
-
const durationMs = Date.now() - startTime;
|
|
140
|
-
logger?.error?.(`Vector sync failed: ${err.message}`);
|
|
141
|
-
return { synced: 0, errors: 1, durationMs };
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
/**
|
|
146
|
-
* Update the local vectors.json index with a new embedding.
|
|
147
|
-
* @param {string} rootDir
|
|
148
|
-
* @param {object} obs - Observation record
|
|
149
|
-
* @param {Float32Array} embedding
|
|
150
|
-
*/
|
|
151
|
-
function updateLocalVectorIndex(rootDir, obs, embedding) {
|
|
152
|
-
const obsDir = join(rootDir, '.cx', 'observations');
|
|
153
|
-
const vectorsPath = join(obsDir, 'vectors.json');
|
|
154
|
-
|
|
155
|
-
let vectors;
|
|
156
|
-
if (existsSync(vectorsPath)) {
|
|
157
|
-
try {
|
|
158
|
-
vectors = JSON.parse(readFileSync(vectorsPath, 'utf8'));
|
|
159
|
-
} catch (err) {
|
|
160
|
-
process.stderr.write('[jobs/vector-sync.mjs] updateLocalVectorIndex: ' + (err?.message ?? String(err)) + '\n');
|
|
161
|
-
vectors = { version: 1, model: 'hashing-bow-v1', updatedAt: null, records: [] };
|
|
162
|
-
}
|
|
163
|
-
} else {
|
|
164
|
-
vectors = { version: 1, model: 'hashing-bow-v1', updatedAt: null, records: [] };
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
// Add or update the record
|
|
168
|
-
const existingIdx = vectors.records.findIndex((r) => r.id === obs.id);
|
|
169
|
-
const record = {
|
|
170
|
-
id: obs.id,
|
|
171
|
-
embedding: Array.from(embedding),
|
|
172
|
-
summary: obs.summary,
|
|
173
|
-
content: obs.content,
|
|
174
|
-
tags: obs.tags || [],
|
|
175
|
-
project: obs.project,
|
|
176
|
-
role: obs.role,
|
|
177
|
-
category: obs.category,
|
|
178
|
-
updatedAt: new Date().toISOString(),
|
|
179
|
-
};
|
|
180
|
-
|
|
181
|
-
if (existingIdx >= 0) {
|
|
182
|
-
vectors.records[existingIdx] = record;
|
|
183
|
-
} else {
|
|
184
|
-
vectors.records.push(record);
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// Cap at 1000 records (sliding window)
|
|
188
|
-
if (vectors.records.length > 1000) {
|
|
189
|
-
vectors.records = vectors.records.slice(-1000);
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
vectors.updatedAt = new Date().toISOString();
|
|
193
|
-
|
|
194
|
-
mkdirSync(dirname(vectorsPath), { recursive: true });
|
|
195
|
-
writeFileSync(vectorsPath, JSON.stringify(vectors, null, 2), 'utf8');
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
export { DEFAULT_BATCH_SIZE, DEFAULT_INTERVAL_MS };
|