@ai-sdlc/orchestrator 0.10.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters.d.ts +42 -3
- package/dist/adapters.js +133 -3
- package/dist/admission-composite.d.ts +112 -1
- package/dist/admission-composite.js +85 -4
- package/dist/admission-enrichment.d.ts +12 -3
- package/dist/admission-enrichment.js +64 -13
- package/dist/admission-score.d.ts +30 -0
- package/dist/admission-score.js +4 -1
- package/dist/analysis/file-walker.js +5 -0
- package/dist/backlog-adapter.d.ts +45 -0
- package/dist/backlog-adapter.js +145 -1
- package/dist/calibration.d.ts +81 -0
- package/dist/calibration.js +76 -0
- package/dist/cli/commands/git-remote.js +1 -1
- package/dist/cli/commands/init-features.d.ts +301 -2
- package/dist/cli/commands/init-features.js +647 -7
- package/dist/cli/commands/init-templates.d.ts +198 -1
- package/dist/cli/commands/init-templates.js +943 -1
- package/dist/cli/commands/init.d.ts +45 -0
- package/dist/cli/commands/init.js +147 -5
- package/dist/cli/commands/run.js +9 -1
- package/dist/cli/index.d.ts +11 -0
- package/dist/cli/index.js +42 -8
- package/dist/compliance/composer.d.ts +79 -0
- package/dist/compliance/composer.js +258 -0
- package/dist/compliance/errors.d.ts +64 -0
- package/dist/compliance/errors.js +85 -0
- package/dist/compliance/loader.d.ts +52 -0
- package/dist/compliance/loader.js +124 -0
- package/dist/compliance/types.d.ts +184 -0
- package/dist/compliance/types.js +41 -0
- package/dist/compliance-clearance.d.ts +269 -0
- package/dist/compliance-clearance.js +269 -0
- package/dist/config.js +17 -0
- package/dist/cost-tracker.d.ts +22 -0
- package/dist/cost-tracker.js +41 -0
- package/dist/cycle-utils.js +14 -1
- package/dist/database/adapters/external.js +5 -1
- package/dist/embedding/adapters/openai-text-embedding-3-small.d.ts +71 -0
- package/dist/embedding/adapters/openai-text-embedding-3-small.js +190 -0
- package/dist/embedding/consumers/tessellation-drift.d.ts +74 -0
- package/dist/embedding/consumers/tessellation-drift.js +76 -0
- package/dist/embedding/cross-provider.d.ts +78 -0
- package/dist/embedding/cross-provider.js +75 -0
- package/dist/embedding/deprecation.d.ts +151 -0
- package/dist/embedding/deprecation.js +229 -0
- package/dist/embedding/errors.d.ts +90 -0
- package/dist/embedding/errors.js +150 -0
- package/dist/embedding/index.d.ts +29 -0
- package/dist/embedding/index.js +24 -0
- package/dist/embedding/pipeline-load.d.ts +146 -0
- package/dist/embedding/pipeline-load.js +178 -0
- package/dist/embedding/registry.d.ts +45 -0
- package/dist/embedding/registry.js +61 -0
- package/dist/embedding/stale-vector.d.ts +110 -0
- package/dist/embedding/stale-vector.js +92 -0
- package/dist/embedding/storage/index.d.ts +51 -0
- package/dist/embedding/storage/index.js +43 -0
- package/dist/embedding/storage/jsonl-backend.d.ts +150 -0
- package/dist/embedding/storage/jsonl-backend.js +332 -0
- package/dist/embedding/storage/types.d.ts +135 -0
- package/dist/embedding/storage/types.js +13 -0
- package/dist/embedding/types.d.ts +180 -0
- package/dist/embedding/types.js +10 -0
- package/dist/execute.d.ts +29 -2
- package/dist/execute.js +200 -38
- package/dist/fix-review.js +1 -1
- package/dist/index.d.ts +14 -2
- package/dist/index.js +19 -2
- package/dist/journey/inheritance-validator.d.ts +396 -0
- package/dist/journey/inheritance-validator.js +370 -0
- package/dist/journey/metric-snapshot.d.ts +403 -0
- package/dist/journey/metric-snapshot.js +370 -0
- package/dist/journey/state-id-drift-rule.d.ts +137 -0
- package/dist/journey/state-id-drift-rule.js +245 -0
- package/dist/journey-sa2-router.d.ts +395 -0
- package/dist/journey-sa2-router.js +308 -0
- package/dist/otel-exporter.js +0 -3
- package/dist/runners/claude-code.js +3 -3
- package/dist/runners/review-agent.js +6 -2
- package/dist/runners/runner-registry.d.ts +36 -0
- package/dist/runners/runner-registry.js +90 -0
- package/dist/runtime/attestations.d.ts +173 -13
- package/dist/runtime/attestations.js +265 -43
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.js +1 -1
- package/dist/sa-scoring/exemplar-bank.js +3 -1
- package/dist/sa-scoring/layer3-llm.js +6 -1
- package/dist/sa-scoring/revision-proposal-config.d.ts +178 -0
- package/dist/sa-scoring/revision-proposal-config.js +198 -0
- package/dist/sa-scoring/revision-proposal.d.ts +285 -0
- package/dist/sa-scoring/revision-proposal.js +417 -0
- package/dist/shared.d.ts +28 -0
- package/dist/shared.js +37 -0
- package/dist/signal-ingestion/adapters/community-thread.d.ts +43 -0
- package/dist/signal-ingestion/adapters/community-thread.js +55 -0
- package/dist/signal-ingestion/adapters/in-app-feedback.d.ts +67 -0
- package/dist/signal-ingestion/adapters/in-app-feedback.js +51 -0
- package/dist/signal-ingestion/adapters/manual.d.ts +78 -0
- package/dist/signal-ingestion/adapters/manual.js +112 -0
- package/dist/signal-ingestion/adapters/support-ticket.d.ts +47 -0
- package/dist/signal-ingestion/adapters/support-ticket.js +51 -0
- package/dist/signal-ingestion/classifier.d.ts +205 -0
- package/dist/signal-ingestion/classifier.js +494 -0
- package/dist/signal-ingestion/clustering-types.d.ts +36 -0
- package/dist/signal-ingestion/clustering-types.js +14 -0
- package/dist/signal-ingestion/clustering.d.ts +200 -0
- package/dist/signal-ingestion/clustering.js +413 -0
- package/dist/signal-ingestion/config.d.ts +351 -0
- package/dist/signal-ingestion/config.js +587 -0
- package/dist/signal-ingestion/d1.d.ts +252 -0
- package/dist/signal-ingestion/d1.js +235 -0
- package/dist/signal-ingestion/errors.d.ts +73 -0
- package/dist/signal-ingestion/errors.js +108 -0
- package/dist/signal-ingestion/governance-events.d.ts +181 -0
- package/dist/signal-ingestion/governance-events.js +189 -0
- package/dist/signal-ingestion/index.d.ts +35 -0
- package/dist/signal-ingestion/index.js +53 -0
- package/dist/signal-ingestion/manual-share-metric.d.ts +93 -0
- package/dist/signal-ingestion/manual-share-metric.js +106 -0
- package/dist/signal-ingestion/registry.d.ts +40 -0
- package/dist/signal-ingestion/registry.js +137 -0
- package/dist/signal-ingestion/residency.d.ts +227 -0
- package/dist/signal-ingestion/residency.js +238 -0
- package/dist/signal-ingestion/significance.d.ts +554 -0
- package/dist/signal-ingestion/significance.js +555 -0
- package/dist/signal-ingestion/types.d.ts +191 -0
- package/dist/signal-ingestion/types.js +8 -0
- package/dist/substrate/drift-composition.d.ts +270 -0
- package/dist/substrate/drift-composition.js +306 -0
- package/dist/substrate/drift-tui-surface.d.ts +61 -0
- package/dist/substrate/drift-tui-surface.js +102 -0
- package/dist/substrate/identity-class.d.ts +176 -0
- package/dist/substrate/identity-class.js +201 -0
- package/dist/tessellation/cross-soul-provenance-rule.d.ts +133 -0
- package/dist/tessellation/cross-soul-provenance-rule.js +171 -0
- package/dist/tessellation/inter-soul-embedding-distance-rule.d.ts +61 -0
- package/dist/tessellation/inter-soul-embedding-distance-rule.js +67 -0
- package/dist/tessellation/rule-registry.d.ts +269 -0
- package/dist/tessellation/rule-registry.js +92 -0
- package/dist/tessellation/soul-slug-ast-scan-rule.d.ts +90 -0
- package/dist/tessellation/soul-slug-ast-scan-rule.js +158 -0
- package/dist/tessellation-admission.d.ts +162 -0
- package/dist/tessellation-admission.js +146 -0
- package/dist/tessellation-drift.d.ts +246 -0
- package/dist/tessellation-drift.js +250 -0
- package/dist/validate-config.js +13 -0
- package/dist/validate-issue.js +2 -2
- package/dist/variant/cardinality-activation.d.ts +126 -0
- package/dist/variant/cardinality-activation.js +101 -0
- package/dist/variant/deprecation-lifecycle.d.ts +184 -0
- package/dist/variant/deprecation-lifecycle.js +208 -0
- package/dist/variant/drift-extension.d.ts +136 -0
- package/dist/variant/drift-extension.js +164 -0
- package/dist/variant/engineering-review.d.ts +185 -0
- package/dist/variant/engineering-review.js +142 -0
- package/dist/variant/index.d.ts +32 -0
- package/dist/variant/index.js +32 -0
- package/dist/variant/inheritance-validator.d.ts +165 -0
- package/dist/variant/inheritance-validator.js +139 -0
- package/dist/variant/internal-adopter/index.d.ts +11 -0
- package/dist/variant/internal-adopter/index.js +10 -0
- package/dist/variant/internal-adopter/products.d.ts +156 -0
- package/dist/variant/internal-adopter/products.js +366 -0
- package/dist/variant-admission.d.ts +316 -0
- package/dist/variant-admission.js +247 -0
- package/dist/webhook-manager.js +7 -1
- package/package.json +15 -12
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC-0030 Phase 3 — Signal clustering.
|
|
3
|
+
*
|
|
4
|
+
* Clusters classified signals into demand themes using one of two algorithms:
|
|
5
|
+
*
|
|
6
|
+
* 1. **BM25 (default)** — pairwise BM25 similarity over signal payloads with
|
|
7
|
+
* greedy single-linkage agglomerative clustering at `similarityThreshold`.
|
|
8
|
+
* Deterministic, model-independent, interpretable. Matches PPA v1.2 Sα₁
|
|
9
|
+
* Layer 2 structural-scoring convention. Requires zero embedding
|
|
10
|
+
* infrastructure — AC #6 graceful-degradation contract.
|
|
11
|
+
*
|
|
12
|
+
* 2. **Embedding (optional)** — when `clustering.algorithm === 'embedding'`
|
|
13
|
+
* AND an `EmbeddingAdapter` (RFC-0019) is supplied via
|
|
14
|
+
* `ClusterSignalsOptions.embeddingAdapter`, the clusterer embeds each
|
|
15
|
+
* signal payload via the adapter and clusters using cosine similarity.
|
|
16
|
+
* Falls back to BM25 with a logged warning when the algorithm is
|
|
17
|
+
* `'embedding'` but no adapter is wired — never silently swaps semantics.
|
|
18
|
+
*
|
|
19
|
+
* **Cluster IDs are deterministic**: SHA-256 of the sorted set of member
|
|
20
|
+
* `sourceId`s, prefixed with `cluster:`. Re-running clustering on the same
|
|
21
|
+
* input produces the same IDs — auditable by construction.
|
|
22
|
+
*
|
|
23
|
+
* **Aggregated cluster metadata** per RFC-0030 §7:
|
|
24
|
+
* - `signalCount`, `uniqueSources`, `oldestSignalAt`, `newestSignalAt`
|
|
25
|
+
* - `icpMatchRate` — fraction of `strong`-resonance member signals
|
|
26
|
+
* - `churnCorrelation` — fraction of `churned`-tier member signals
|
|
27
|
+
* - `tier1SignalCount` / `tier2SignalCount` — counts by adapter tier, for
|
|
28
|
+
* Phase 4's Tier 2 significance threshold consumer
|
|
29
|
+
* - `aggregateRecencyDecay` — mean recency decay across members
|
|
30
|
+
*
|
|
31
|
+
* **Out of scope for Phase 3** (intentional):
|
|
32
|
+
* - `saResonance` — left undefined; computed in Phase 4 against the Soul DID
|
|
33
|
+
* - `topSummary` — left undefined; cluster summarisation is post-MVP
|
|
34
|
+
*
|
|
35
|
+
* @module signal-ingestion/clustering
|
|
36
|
+
*/
|
|
37
|
+
import type { EmbeddingAdapter } from '../embedding/types.js';
|
|
38
|
+
import type { ClusteredSignalInput } from './clustering-types.js';
|
|
39
|
+
import type { SignalIngestionConfig } from './config.js';
|
|
40
|
+
import type { CustomerTier, RawSignal, SignalTier } from './types.js';
|
|
41
|
+
export type { ClusteredSignalInput } from './clustering-types.js';
|
|
42
|
+
/**
|
|
43
|
+
* Algorithm that was actually used to produce a clustering result.
|
|
44
|
+
* Exposed for observability — operators can audit whether embedding was
|
|
45
|
+
* configured but fell back to BM25 because no adapter was wired.
|
|
46
|
+
*/
|
|
47
|
+
export type ClusteringAlgorithmUsed = 'bm25' | 'embedding';
|
|
48
|
+
/**
|
|
49
|
+
* Aggregated demand cluster output per RFC-0030 §7.
|
|
50
|
+
*
|
|
51
|
+
* Member signals are stored as `ClusteredSignalInput` (the same shape the
|
|
52
|
+
* clusterer receives) so downstream Phase 4/5 consumers retain access to the
|
|
53
|
+
* classification axes (tier, ICP resonance, recency decay) without re-running
|
|
54
|
+
* the classifier.
|
|
55
|
+
*/
|
|
56
|
+
export interface DemandCluster {
|
|
57
|
+
/** Deterministic SHA-256-derived ID; prefixed with `cluster:`. */
|
|
58
|
+
clusterId: string;
|
|
59
|
+
/** Member signals in input order. */
|
|
60
|
+
members: ClusteredSignalInput[];
|
|
61
|
+
/** Total member count. */
|
|
62
|
+
signalCount: number;
|
|
63
|
+
/** Number of distinct source adapter names across members. */
|
|
64
|
+
uniqueSources: number;
|
|
65
|
+
/** Number of Tier 1 (direct) signals in the cluster. */
|
|
66
|
+
tier1SignalCount: number;
|
|
67
|
+
/** Number of Tier 2 (ambient) signals in the cluster. */
|
|
68
|
+
tier2SignalCount: number;
|
|
69
|
+
/** Oldest member `sourceTimestamp`. */
|
|
70
|
+
oldestSignalAt: Date;
|
|
71
|
+
/** Newest member `sourceTimestamp`. */
|
|
72
|
+
newestSignalAt: Date;
|
|
73
|
+
/** Fraction of members with `icpResonance === 'strong'`. Range [0, 1]. */
|
|
74
|
+
icpMatchRate: number;
|
|
75
|
+
/** Fraction of members with `customerTier === 'churned'`. Range [0, 1]. */
|
|
76
|
+
churnCorrelation: number;
|
|
77
|
+
/** Mean recency decay across members. Range [0, 1]. */
|
|
78
|
+
aggregateRecencyDecay: number;
|
|
79
|
+
/**
|
|
80
|
+
* SA resonance against the Soul DID — computed in Phase 4 (RFC-0030 §9).
|
|
81
|
+
* Always `undefined` from Phase 3.
|
|
82
|
+
*/
|
|
83
|
+
saResonance?: number;
|
|
84
|
+
/**
|
|
85
|
+
* Cluster-theme summary — generated by LLM or adapter in a future phase.
|
|
86
|
+
* Always `undefined` from Phase 3.
|
|
87
|
+
*/
|
|
88
|
+
topSummary?: string;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Result of clustering a batch of classified signals.
|
|
92
|
+
*/
|
|
93
|
+
export interface ClusteringResult {
|
|
94
|
+
clusters: DemandCluster[];
|
|
95
|
+
/** Algorithm actually used (may differ from config when adapter missing). */
|
|
96
|
+
algorithmUsed: ClusteringAlgorithmUsed;
|
|
97
|
+
/**
|
|
98
|
+
* Set when `clustering.algorithm === 'embedding'` but no adapter was wired
|
|
99
|
+
* (graceful fallback to BM25 per AC #6).
|
|
100
|
+
*/
|
|
101
|
+
fallbackReason?: 'embedding-adapter-missing' | 'embedding-adapter-unavailable';
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Options for `clusterSignals()`.
|
|
105
|
+
*/
|
|
106
|
+
export interface ClusterSignalsOptions {
|
|
107
|
+
/**
|
|
108
|
+
* Resolved signal ingestion configuration. Defaults to the framework
|
|
109
|
+
* default (`bm25` + `similarityThreshold: 0.6`).
|
|
110
|
+
*/
|
|
111
|
+
config?: SignalIngestionConfig;
|
|
112
|
+
/**
|
|
113
|
+
* Embedding adapter for the optional `embedding` algorithm. Resolved per
|
|
114
|
+
* RFC-0019 § registry semantics by the caller (read the `embedding-config.yaml`
|
|
115
|
+
* provider field; pass the resolved adapter here). Ignored when the
|
|
116
|
+
* configured algorithm is `bm25`. When the configured algorithm is
|
|
117
|
+
* `embedding` but this field is `undefined`, the clusterer logs a fallback
|
|
118
|
+
* and uses BM25 (graceful degradation per AC #6).
|
|
119
|
+
*/
|
|
120
|
+
embeddingAdapter?: EmbeddingAdapter;
|
|
121
|
+
/**
|
|
122
|
+
* Optional override of the consumer label sent to the embedding adapter
|
|
123
|
+
* for cost attribution. Defaults to `'rfc-0030-clustering'`.
|
|
124
|
+
*/
|
|
125
|
+
embeddingConsumerLabel?: string;
|
|
126
|
+
/**
|
|
127
|
+
* Optional hook invoked when the clusterer falls back from embedding to
|
|
128
|
+
* BM25. Defaults to a no-op (callers wire console.warn / structured logger).
|
|
129
|
+
*/
|
|
130
|
+
onFallback?: (info: {
|
|
131
|
+
reason: 'embedding-adapter-missing' | 'embedding-adapter-unavailable';
|
|
132
|
+
message: string;
|
|
133
|
+
}) => void;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Cluster a batch of classified signals into demand themes.
|
|
137
|
+
*
|
|
138
|
+
* Algorithm selection (per `config.clustering.algorithm`):
|
|
139
|
+
* - `'bm25'` (default) — pairwise BM25 similarity, greedy single-linkage.
|
|
140
|
+
* - `'embedding'` — requires `options.embeddingAdapter`; falls back to BM25
|
|
141
|
+
* with `fallbackReason: 'embedding-adapter-missing'` when not supplied
|
|
142
|
+
* (AC #6 graceful degradation).
|
|
143
|
+
*
|
|
144
|
+
* Threshold: `config.clustering.similarityThreshold` (default 0.6).
|
|
145
|
+
*
|
|
146
|
+
* Cluster IDs are deterministic — re-running with the same inputs yields the
|
|
147
|
+
* same IDs by SHA-256 of the sorted member `sourceId` set.
|
|
148
|
+
*/
|
|
149
|
+
export declare function clusterSignals(signals: ClusteredSignalInput[], options?: ClusterSignalsOptions): Promise<ClusteringResult>;
|
|
150
|
+
/**
|
|
151
|
+
* Cosine similarity between two equal-length numeric vectors.
|
|
152
|
+
* Returns 0 when either vector has zero magnitude.
|
|
153
|
+
*/
|
|
154
|
+
export declare function cosineSimilarity(a: number[], b: number[]): number;
|
|
155
|
+
/**
|
|
156
|
+
* Compute a deterministic SHA-256-derived cluster ID from the sorted set of
|
|
157
|
+
* member sourceIds. The first 24 hex chars of the digest are appended to a
|
|
158
|
+
* `cluster:` prefix — collision-resistant within any realistic ingestion
|
|
159
|
+
* window while remaining compact for log/event readability.
|
|
160
|
+
*/
|
|
161
|
+
export declare function computeClusterId(sortedSourceIds: string[]): string;
|
|
162
|
+
/**
|
|
163
|
+
* Cluster signals with per-region segregation per RFC-0030 v0.3 OQ-13.3
|
|
164
|
+
* clustering enforcement point. Partitions the input by `signal.region`
|
|
165
|
+
* (via `partitionSignalsByRegion`) and runs `clusterSignals()` once per
|
|
166
|
+
* partition, concatenating the per-partition `DemandCluster[]` outputs so
|
|
167
|
+
* cross-region cluster merge is structurally impossible.
|
|
168
|
+
*
|
|
169
|
+
* **Decision boundary**: when `partitionByRegion` is `false` (caller-supplied
|
|
170
|
+
* or the active regime doesn't require segregation via
|
|
171
|
+
* `clusterRequiresSegregation()`), this function falls through to a single
|
|
172
|
+
* `clusterSignals()` call — zero overhead for adopters with no residency
|
|
173
|
+
* regime declared.
|
|
174
|
+
*
|
|
175
|
+
* The returned `ClusteringResult.algorithmUsed` is the algorithm used by the
|
|
176
|
+
* FIRST partition (all partitions use the same algorithm; `fallbackReason`
|
|
177
|
+
* surfaces if any partition fell back from embedding to BM25). When
|
|
178
|
+
* `partitionByRegion: true`, the function additionally returns
|
|
179
|
+
* `regionPartitions: { region: clusterCount }` for audit / cost-attribution
|
|
180
|
+
* surfacing.
|
|
181
|
+
*/
|
|
182
|
+
export interface ClusterSignalsWithResidencyOptions extends ClusterSignalsOptions {
|
|
183
|
+
/**
|
|
184
|
+
* When true, signals are partitioned by `region` before similarity
|
|
185
|
+
* computation. Defaults to `false` (caller decides based on the active
|
|
186
|
+
* regime declaration via `clusterRequiresSegregation()`).
|
|
187
|
+
*/
|
|
188
|
+
partitionByRegion?: boolean;
|
|
189
|
+
}
|
|
190
|
+
export interface ClusteringResultWithResidency extends ClusteringResult {
|
|
191
|
+
/**
|
|
192
|
+
* Per-region cluster count breakdown. Only populated when
|
|
193
|
+
* `partitionByRegion: true`. Region keys lowercased; `__unspecified` is the
|
|
194
|
+
* bucket for signals with no `region` tag.
|
|
195
|
+
*/
|
|
196
|
+
regionPartitions?: Record<string, number>;
|
|
197
|
+
}
|
|
198
|
+
export declare function clusterSignalsWithResidency(signals: ClusteredSignalInput[], options?: ClusterSignalsWithResidencyOptions): Promise<ClusteringResultWithResidency>;
|
|
199
|
+
export type { CustomerTier, RawSignal, SignalTier };
|
|
200
|
+
//# sourceMappingURL=clustering.d.ts.map
|
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC-0030 Phase 3 — Signal clustering.
|
|
3
|
+
*
|
|
4
|
+
* Clusters classified signals into demand themes using one of two algorithms:
|
|
5
|
+
*
|
|
6
|
+
* 1. **BM25 (default)** — pairwise BM25 similarity over signal payloads with
|
|
7
|
+
* greedy single-linkage agglomerative clustering at `similarityThreshold`.
|
|
8
|
+
* Deterministic, model-independent, interpretable. Matches PPA v1.2 Sα₁
|
|
9
|
+
* Layer 2 structural-scoring convention. Requires zero embedding
|
|
10
|
+
* infrastructure — AC #6 graceful-degradation contract.
|
|
11
|
+
*
|
|
12
|
+
* 2. **Embedding (optional)** — when `clustering.algorithm === 'embedding'`
|
|
13
|
+
* AND an `EmbeddingAdapter` (RFC-0019) is supplied via
|
|
14
|
+
* `ClusterSignalsOptions.embeddingAdapter`, the clusterer embeds each
|
|
15
|
+
* signal payload via the adapter and clusters using cosine similarity.
|
|
16
|
+
* Falls back to BM25 with a logged warning when the algorithm is
|
|
17
|
+
* `'embedding'` but no adapter is wired — never silently swaps semantics.
|
|
18
|
+
*
|
|
19
|
+
* **Cluster IDs are deterministic**: SHA-256 of the sorted set of member
|
|
20
|
+
* `sourceId`s, prefixed with `cluster:`. Re-running clustering on the same
|
|
21
|
+
* input produces the same IDs — auditable by construction.
|
|
22
|
+
*
|
|
23
|
+
* **Aggregated cluster metadata** per RFC-0030 §7:
|
|
24
|
+
* - `signalCount`, `uniqueSources`, `oldestSignalAt`, `newestSignalAt`
|
|
25
|
+
* - `icpMatchRate` — fraction of `strong`-resonance member signals
|
|
26
|
+
* - `churnCorrelation` — fraction of `churned`-tier member signals
|
|
27
|
+
* - `tier1SignalCount` / `tier2SignalCount` — counts by adapter tier, for
|
|
28
|
+
* Phase 4's Tier 2 significance threshold consumer
|
|
29
|
+
* - `aggregateRecencyDecay` — mean recency decay across members
|
|
30
|
+
*
|
|
31
|
+
* **Out of scope for Phase 3** (intentional):
|
|
32
|
+
* - `saResonance` — left undefined; computed in Phase 4 against the Soul DID
|
|
33
|
+
* - `topSummary` — left undefined; cluster summarisation is post-MVP
|
|
34
|
+
*
|
|
35
|
+
* @module signal-ingestion/clustering
|
|
36
|
+
*/
|
|
37
|
+
import { createHash } from 'node:crypto';
|
|
38
|
+
import { DEFAULT_SIGNAL_INGESTION_CONFIG } from './config.js';
|
|
39
|
+
import { tokenize } from './classifier.js';
|
|
40
|
+
// ── Main entry point ────────────────────────────────────────────────────────
|
|
41
|
+
/**
|
|
42
|
+
* Cluster a batch of classified signals into demand themes.
|
|
43
|
+
*
|
|
44
|
+
* Algorithm selection (per `config.clustering.algorithm`):
|
|
45
|
+
* - `'bm25'` (default) — pairwise BM25 similarity, greedy single-linkage.
|
|
46
|
+
* - `'embedding'` — requires `options.embeddingAdapter`; falls back to BM25
|
|
47
|
+
* with `fallbackReason: 'embedding-adapter-missing'` when not supplied
|
|
48
|
+
* (AC #6 graceful degradation).
|
|
49
|
+
*
|
|
50
|
+
* Threshold: `config.clustering.similarityThreshold` (default 0.6).
|
|
51
|
+
*
|
|
52
|
+
* Cluster IDs are deterministic — re-running with the same inputs yields the
|
|
53
|
+
* same IDs by SHA-256 of the sorted member `sourceId` set.
|
|
54
|
+
*/
|
|
55
|
+
export async function clusterSignals(signals, options = {}) {
|
|
56
|
+
const config = options.config ?? DEFAULT_SIGNAL_INGESTION_CONFIG;
|
|
57
|
+
const clustering = config.clustering;
|
|
58
|
+
const threshold = clustering.similarityThreshold;
|
|
59
|
+
if (signals.length === 0) {
|
|
60
|
+
return { clusters: [], algorithmUsed: clustering.algorithm };
|
|
61
|
+
}
|
|
62
|
+
let algorithmUsed = clustering.algorithm;
|
|
63
|
+
let fallbackReason;
|
|
64
|
+
// Resolve algorithm — graceful fallback when embedding is configured but
|
|
65
|
+
// no adapter is wired (AC #6).
|
|
66
|
+
let similarityMatrix;
|
|
67
|
+
if (clustering.algorithm === 'embedding') {
|
|
68
|
+
if (!options.embeddingAdapter) {
|
|
69
|
+
algorithmUsed = 'bm25';
|
|
70
|
+
fallbackReason = 'embedding-adapter-missing';
|
|
71
|
+
options.onFallback?.({
|
|
72
|
+
reason: 'embedding-adapter-missing',
|
|
73
|
+
message: "clustering.algorithm='embedding' configured but no embeddingAdapter supplied; falling back to BM25",
|
|
74
|
+
});
|
|
75
|
+
similarityMatrix = computeBm25Matrix(signals);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
const availability = await options.embeddingAdapter.isAvailable();
|
|
79
|
+
if (!availability.available) {
|
|
80
|
+
algorithmUsed = 'bm25';
|
|
81
|
+
fallbackReason = 'embedding-adapter-unavailable';
|
|
82
|
+
options.onFallback?.({
|
|
83
|
+
reason: 'embedding-adapter-unavailable',
|
|
84
|
+
message: `embedding adapter '${options.embeddingAdapter.name}' is unavailable (${availability.reason ?? 'unknown'}); falling back to BM25`,
|
|
85
|
+
});
|
|
86
|
+
similarityMatrix = computeBm25Matrix(signals);
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
similarityMatrix = await computeEmbeddingMatrix(signals, options.embeddingAdapter, options.embeddingConsumerLabel ?? 'rfc-0030-clustering');
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
similarityMatrix = computeBm25Matrix(signals);
|
|
95
|
+
}
|
|
96
|
+
const clusterMembership = singleLinkageCluster(signals.length, similarityMatrix, threshold);
|
|
97
|
+
const clusters = buildDemandClusters(signals, clusterMembership);
|
|
98
|
+
return { clusters, algorithmUsed, fallbackReason };
|
|
99
|
+
}
|
|
100
|
+
// ── BM25 similarity matrix ──────────────────────────────────────────────────
|
|
101
|
+
/**
|
|
102
|
+
* Compute pairwise normalised BM25 similarity over signal payloads.
|
|
103
|
+
*
|
|
104
|
+
* BM25 (k1=1.5, b=0.75) is the standard PPA Sα₁ Layer 2 scoring choice. We
|
|
105
|
+
* normalise each pairwise score by the self-score of the shorter document so
|
|
106
|
+
* the result lives in [0, 1] and is symmetric around the diagonal.
|
|
107
|
+
*
|
|
108
|
+
* Returns an N×N matrix; cell [i][j] is the symmetric similarity. The
|
|
109
|
+
* diagonal is set to 1.0.
|
|
110
|
+
*/
|
|
111
|
+
function computeBm25Matrix(signals) {
|
|
112
|
+
const N = signals.length;
|
|
113
|
+
const K1 = 1.5;
|
|
114
|
+
const B = 0.75;
|
|
115
|
+
const tokens = signals.map((s) => tokenize(s.signal.payload));
|
|
116
|
+
const tfs = tokens.map((toks) => termFrequency(toks));
|
|
117
|
+
const docLens = tokens.map((toks) => toks.length);
|
|
118
|
+
const totalLen = docLens.reduce((a, b) => a + b, 0);
|
|
119
|
+
const avgDocLen = N > 0 ? totalLen / N : 0;
|
|
120
|
+
// Pre-compute IDF per term across the corpus (collection of payloads).
|
|
121
|
+
const allTerms = new Set();
|
|
122
|
+
for (const tf of tfs)
|
|
123
|
+
for (const term of tf.keys())
|
|
124
|
+
allTerms.add(term);
|
|
125
|
+
const idfMap = new Map();
|
|
126
|
+
for (const term of allTerms) {
|
|
127
|
+
let df = 0;
|
|
128
|
+
for (const tf of tfs)
|
|
129
|
+
if ((tf.get(term) ?? 0) > 0)
|
|
130
|
+
df++;
|
|
131
|
+
idfMap.set(term, Math.log((N - df + 0.5) / (df + 0.5) + 1));
|
|
132
|
+
}
|
|
133
|
+
const scoreDocWithQuery = (docIdx, queryTerms) => {
|
|
134
|
+
const tf = tfs[docIdx];
|
|
135
|
+
const docLen = docLens[docIdx];
|
|
136
|
+
let score = 0;
|
|
137
|
+
for (const term of new Set(queryTerms)) {
|
|
138
|
+
const freq = tf.get(term) ?? 0;
|
|
139
|
+
if (freq === 0)
|
|
140
|
+
continue;
|
|
141
|
+
const termIdf = idfMap.get(term) ?? 0;
|
|
142
|
+
const numerator = freq * (K1 + 1);
|
|
143
|
+
const denominator = freq + K1 * (1 - B + B * (docLen / Math.max(avgDocLen, 1)));
|
|
144
|
+
score += termIdf * (numerator / denominator);
|
|
145
|
+
}
|
|
146
|
+
return score;
|
|
147
|
+
};
|
|
148
|
+
// Self-scores for normalisation.
|
|
149
|
+
const selfScores = signals.map((_, i) => scoreDocWithQuery(i, tokens[i]));
|
|
150
|
+
const matrix = Array.from({ length: N }, () => new Array(N).fill(0));
|
|
151
|
+
for (let i = 0; i < N; i++) {
|
|
152
|
+
matrix[i][i] = 1.0;
|
|
153
|
+
for (let j = i + 1; j < N; j++) {
|
|
154
|
+
// Score doc j with doc i's terms, doc i with doc j's terms.
|
|
155
|
+
const sij = scoreDocWithQuery(j, tokens[i]);
|
|
156
|
+
const sji = scoreDocWithQuery(i, tokens[j]);
|
|
157
|
+
// Normalise each direction by the queried doc's self-score; then average
|
|
158
|
+
// for symmetry. Avoids division-by-zero when a payload is empty.
|
|
159
|
+
const normIJ = selfScores[j] > 0 ? sij / selfScores[j] : 0;
|
|
160
|
+
const normJI = selfScores[i] > 0 ? sji / selfScores[i] : 0;
|
|
161
|
+
const sim = Math.min(1, (normIJ + normJI) / 2);
|
|
162
|
+
matrix[i][j] = sim;
|
|
163
|
+
matrix[j][i] = sim;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return matrix;
|
|
167
|
+
}
|
|
168
|
+
function termFrequency(tokens) {
|
|
169
|
+
const tf = new Map();
|
|
170
|
+
for (const t of tokens)
|
|
171
|
+
tf.set(t, (tf.get(t) ?? 0) + 1);
|
|
172
|
+
return tf;
|
|
173
|
+
}
|
|
174
|
+
// ── Embedding similarity matrix ─────────────────────────────────────────────
|
|
175
|
+
/**
|
|
176
|
+
* Embed each signal payload via the supplied adapter and compute pairwise
|
|
177
|
+
* cosine similarity.
|
|
178
|
+
*
|
|
179
|
+
* Uses `embedBatch()` when available for efficiency; falls back to a sequential
|
|
180
|
+
* `embed()` loop otherwise. Per-call cost is attributed to `consumerLabel`
|
|
181
|
+
* (default `'rfc-0030-clustering'`) per RFC-0019 OQ-6.
|
|
182
|
+
*/
|
|
183
|
+
async function computeEmbeddingMatrix(signals, adapter, consumerLabel) {
|
|
184
|
+
const N = signals.length;
|
|
185
|
+
const payloads = signals.map((s) => s.signal.payload);
|
|
186
|
+
let vectors;
|
|
187
|
+
if (adapter.embedBatch) {
|
|
188
|
+
vectors = await adapter.embedBatch(payloads, consumerLabel);
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
vectors = [];
|
|
192
|
+
for (const text of payloads) {
|
|
193
|
+
vectors.push(await adapter.embed(text, consumerLabel));
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
const matrix = Array.from({ length: N }, () => new Array(N).fill(0));
|
|
197
|
+
for (let i = 0; i < N; i++) {
|
|
198
|
+
matrix[i][i] = 1.0;
|
|
199
|
+
for (let j = i + 1; j < N; j++) {
|
|
200
|
+
const sim = cosineSimilarity(vectors[i], vectors[j]);
|
|
201
|
+
matrix[i][j] = sim;
|
|
202
|
+
matrix[j][i] = sim;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return matrix;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Cosine similarity between two equal-length numeric vectors.
|
|
209
|
+
* Returns 0 when either vector has zero magnitude.
|
|
210
|
+
*/
|
|
211
|
+
export function cosineSimilarity(a, b) {
|
|
212
|
+
if (a.length !== b.length) {
|
|
213
|
+
throw new Error(`cosineSimilarity: vector length mismatch (${a.length} vs ${b.length})`);
|
|
214
|
+
}
|
|
215
|
+
let dot = 0;
|
|
216
|
+
let normA = 0;
|
|
217
|
+
let normB = 0;
|
|
218
|
+
for (let i = 0; i < a.length; i++) {
|
|
219
|
+
const va = a[i];
|
|
220
|
+
const vb = b[i];
|
|
221
|
+
dot += va * vb;
|
|
222
|
+
normA += va * va;
|
|
223
|
+
normB += vb * vb;
|
|
224
|
+
}
|
|
225
|
+
if (normA === 0 || normB === 0)
|
|
226
|
+
return 0;
|
|
227
|
+
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
228
|
+
}
|
|
229
|
+
// ── Single-linkage agglomerative clustering ─────────────────────────────────
|
|
230
|
+
/**
|
|
231
|
+
* Greedy single-linkage agglomerative clustering using Union-Find.
|
|
232
|
+
*
|
|
233
|
+
* Two signals join the same cluster whenever their pairwise similarity meets
|
|
234
|
+
* or exceeds `threshold`. This is the simplest correct connect-the-dots
|
|
235
|
+
* algorithm: stable, deterministic, no parameters to tune beyond `threshold`,
|
|
236
|
+
* O(N^2) which is acceptable for the v1 ingestion window (operators typically
|
|
237
|
+
* cluster batches of 10²–10³ signals per pipeline-load).
|
|
238
|
+
*
|
|
239
|
+
* Returns an array `parent[i]` giving the cluster representative for signal i.
|
|
240
|
+
*/
|
|
241
|
+
function singleLinkageCluster(n, sim, threshold) {
|
|
242
|
+
const parent = Array.from({ length: n }, (_, i) => i);
|
|
243
|
+
const find = (x) => {
|
|
244
|
+
let root = x;
|
|
245
|
+
while (parent[root] !== root)
|
|
246
|
+
root = parent[root];
|
|
247
|
+
// Path compression.
|
|
248
|
+
let cur = x;
|
|
249
|
+
while (parent[cur] !== root) {
|
|
250
|
+
const next = parent[cur];
|
|
251
|
+
parent[cur] = root;
|
|
252
|
+
cur = next;
|
|
253
|
+
}
|
|
254
|
+
return root;
|
|
255
|
+
};
|
|
256
|
+
const union = (a, b) => {
|
|
257
|
+
const ra = find(a);
|
|
258
|
+
const rb = find(b);
|
|
259
|
+
if (ra === rb)
|
|
260
|
+
return;
|
|
261
|
+
// Union by lower index — keeps cluster representative deterministic.
|
|
262
|
+
if (ra < rb)
|
|
263
|
+
parent[rb] = ra;
|
|
264
|
+
else
|
|
265
|
+
parent[ra] = rb;
|
|
266
|
+
};
|
|
267
|
+
for (let i = 0; i < n; i++) {
|
|
268
|
+
for (let j = i + 1; j < n; j++) {
|
|
269
|
+
if (sim[i][j] >= threshold) {
|
|
270
|
+
union(i, j);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
return parent.map((_, i) => find(i));
|
|
275
|
+
}
|
|
276
|
+
// ── DemandCluster assembly ──────────────────────────────────────────────────
|
|
277
|
+
/**
|
|
278
|
+
* Assemble `DemandCluster` records from per-signal membership.
|
|
279
|
+
* Aggregates tier/ICP/recency/timestamp metadata + computes deterministic IDs.
|
|
280
|
+
*
|
|
281
|
+
* Output ordering: clusters are sorted by `clusterId` (sha256-derived hex,
|
|
282
|
+
* lexicographic) so the result is deterministic regardless of input order.
|
|
283
|
+
*/
|
|
284
|
+
function buildDemandClusters(signals, membership) {
|
|
285
|
+
const byRoot = new Map();
|
|
286
|
+
for (let i = 0; i < signals.length; i++) {
|
|
287
|
+
const root = membership[i];
|
|
288
|
+
let bucket = byRoot.get(root);
|
|
289
|
+
if (!bucket) {
|
|
290
|
+
bucket = [];
|
|
291
|
+
byRoot.set(root, bucket);
|
|
292
|
+
}
|
|
293
|
+
bucket.push(signals[i]);
|
|
294
|
+
}
|
|
295
|
+
const clusters = [];
|
|
296
|
+
for (const members of byRoot.values()) {
|
|
297
|
+
clusters.push(assembleCluster(members));
|
|
298
|
+
}
|
|
299
|
+
// Sort clusters by deterministic clusterId for stable output ordering.
|
|
300
|
+
clusters.sort((a, b) => a.clusterId.localeCompare(b.clusterId));
|
|
301
|
+
return clusters;
|
|
302
|
+
}
|
|
303
|
+
function assembleCluster(members) {
|
|
304
|
+
const sourceIds = members.map((m) => m.signal.sourceId).sort();
|
|
305
|
+
const clusterId = computeClusterId(sourceIds);
|
|
306
|
+
let oldestMs = Infinity;
|
|
307
|
+
let newestMs = -Infinity;
|
|
308
|
+
let strongCount = 0;
|
|
309
|
+
let churnedCount = 0;
|
|
310
|
+
let tier1 = 0;
|
|
311
|
+
let tier2 = 0;
|
|
312
|
+
let recencySum = 0;
|
|
313
|
+
const adapterNames = new Set();
|
|
314
|
+
for (const m of members) {
|
|
315
|
+
const ts = m.signal.sourceTimestamp.getTime();
|
|
316
|
+
if (ts < oldestMs)
|
|
317
|
+
oldestMs = ts;
|
|
318
|
+
if (ts > newestMs)
|
|
319
|
+
newestMs = ts;
|
|
320
|
+
if (m.icpResonance === 'strong')
|
|
321
|
+
strongCount++;
|
|
322
|
+
if (m.customerTier === 'churned')
|
|
323
|
+
churnedCount++;
|
|
324
|
+
const adapterTier = resolveAdapterTier(m.signal, m.adapterTier);
|
|
325
|
+
if (adapterTier === 1)
|
|
326
|
+
tier1++;
|
|
327
|
+
else
|
|
328
|
+
tier2++;
|
|
329
|
+
recencySum += m.recencyDecay;
|
|
330
|
+
const adapterName = m.signal.metadata?.['adapterName'] ?? 'unknown';
|
|
331
|
+
adapterNames.add(adapterName);
|
|
332
|
+
}
|
|
333
|
+
const n = members.length;
|
|
334
|
+
return {
|
|
335
|
+
clusterId,
|
|
336
|
+
members: [...members],
|
|
337
|
+
signalCount: n,
|
|
338
|
+
uniqueSources: adapterNames.size,
|
|
339
|
+
tier1SignalCount: tier1,
|
|
340
|
+
tier2SignalCount: tier2,
|
|
341
|
+
oldestSignalAt: new Date(oldestMs),
|
|
342
|
+
newestSignalAt: new Date(newestMs),
|
|
343
|
+
icpMatchRate: strongCount / n,
|
|
344
|
+
churnCorrelation: churnedCount / n,
|
|
345
|
+
aggregateRecencyDecay: recencySum / n,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
/**
|
|
349
|
+
* Resolve the adapter tier for a member signal. Prefers the explicit
|
|
350
|
+
* `adapterTier` field on the input (most-authoritative); falls back to
|
|
351
|
+
* `signal.metadata.adapterTier` for adapters that surface the value via
|
|
352
|
+
* metadata; defaults to Tier 1 (matches the classifier convention).
|
|
353
|
+
*/
|
|
354
|
+
function resolveAdapterTier(signal, explicit) {
|
|
355
|
+
if (explicit !== undefined)
|
|
356
|
+
return explicit;
|
|
357
|
+
const fromMetadata = signal.metadata?.['adapterTier'];
|
|
358
|
+
if (fromMetadata === 1 || fromMetadata === 2)
|
|
359
|
+
return fromMetadata;
|
|
360
|
+
return 1;
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Compute a deterministic SHA-256-derived cluster ID from the sorted set of
|
|
364
|
+
* member sourceIds. The first 24 hex chars of the digest are appended to a
|
|
365
|
+
* `cluster:` prefix — collision-resistant within any realistic ingestion
|
|
366
|
+
* window while remaining compact for log/event readability.
|
|
367
|
+
*/
|
|
368
|
+
export function computeClusterId(sortedSourceIds) {
|
|
369
|
+
const hash = createHash('sha256');
|
|
370
|
+
for (const id of sortedSourceIds) {
|
|
371
|
+
hash.update(id);
|
|
372
|
+
hash.update(''); // null-byte delimiter; ensures ['a', 'bc'] !== ['ab', 'c']
|
|
373
|
+
}
|
|
374
|
+
return `cluster:${hash.digest('hex').slice(0, 24)}`;
|
|
375
|
+
}
|
|
376
|
+
export async function clusterSignalsWithResidency(signals, options = {}) {
|
|
377
|
+
const partitionByRegion = options.partitionByRegion ?? false;
|
|
378
|
+
if (!partitionByRegion) {
|
|
379
|
+
return clusterSignals(signals, options);
|
|
380
|
+
}
|
|
381
|
+
// Lazy-import to avoid a clustering -> residency import cycle.
|
|
382
|
+
const { partitionSignalsByRegion } = await import('./residency.js');
|
|
383
|
+
const partitions = partitionSignalsByRegion(signals);
|
|
384
|
+
const allClusters = [];
|
|
385
|
+
const regionPartitions = {};
|
|
386
|
+
let algorithmUsed = 'bm25';
|
|
387
|
+
let fallbackReason;
|
|
388
|
+
for (const [region, partitionSignals] of partitions) {
|
|
389
|
+
if (partitionSignals.length === 0)
|
|
390
|
+
continue;
|
|
391
|
+
// Strip partitionByRegion from options when delegating so we don't
|
|
392
|
+
// recurse (clusterSignals doesn't read this field anyway, but defensive).
|
|
393
|
+
const { partitionByRegion: _ignored, ...delegateOptions } = options;
|
|
394
|
+
const result = await clusterSignals(partitionSignals, delegateOptions);
|
|
395
|
+
allClusters.push(...result.clusters);
|
|
396
|
+
regionPartitions[region] = result.clusters.length;
|
|
397
|
+
algorithmUsed = result.algorithmUsed;
|
|
398
|
+
if (result.fallbackReason !== undefined)
|
|
399
|
+
fallbackReason = result.fallbackReason;
|
|
400
|
+
}
|
|
401
|
+
// Sort across all partitions by deterministic clusterId so the output
|
|
402
|
+
// ordering matches non-partitioned clusterSignals semantics.
|
|
403
|
+
allClusters.sort((a, b) => a.clusterId.localeCompare(b.clusterId));
|
|
404
|
+
const out = {
|
|
405
|
+
clusters: allClusters,
|
|
406
|
+
algorithmUsed,
|
|
407
|
+
regionPartitions,
|
|
408
|
+
};
|
|
409
|
+
if (fallbackReason !== undefined)
|
|
410
|
+
out.fallbackReason = fallbackReason;
|
|
411
|
+
return out;
|
|
412
|
+
}
|
|
413
|
+
//# sourceMappingURL=clustering.js.map
|