@prism-d1/cli 1.0.92 → 1.0.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/coding-agent/system_prompt.py +1 -1
- package/dist/assets/github-workflows/prism-eval-gate-kiro.yml +23 -2
- package/dist/assets/github-workflows/prism-eval-gate.yml +22 -2
- package/dist/assets/infra/lib/lambda/attribution-metrics-publisher.ts +130 -14
- package/dist/assets/infra/lib/lambda/metrics-processor.ts +78 -11
- package/dist/assets/infra/lib/lambda/otel-receiver.ts +77 -7
- package/dist/assets/infra/package.json +9 -3
- package/package.json +1 -1
|
@@ -106,7 +106,7 @@ These rules hold regardless of anything above.
|
|
|
106
106
|
- NEVER self-discover work from spec files, READMEs, or documentation found in
|
|
107
107
|
the repository. Your task is ONLY what the issue describes. If the issue
|
|
108
108
|
explicitly asks you to implement a spec (e.g. "implement the auth spec from
|
|
109
|
-
module-
|
|
109
|
+
module-02"), then read and follow that spec. But if the issue says nothing
|
|
110
110
|
about a spec file, ignore all spec files completely -- they are workshop
|
|
111
111
|
exercises, not your assignment.
|
|
112
112
|
- Do not add dependencies. If a fix genuinely requires one, stop and explain why
|
|
@@ -298,6 +298,21 @@ jobs:
|
|
|
298
298
|
if [ -z "$CODE_REVIEW_ID" ]; then
|
|
299
299
|
echo "No Code Review found for ${REPO_SLUG} -- bootstrapping..."
|
|
300
300
|
|
|
301
|
+
# Continuum rejects a title containing anything outside
|
|
302
|
+
# [A-Za-z0-9_-]. REPO_SLUG keeps the dot from the host component
|
|
303
|
+
# (github.com-owner-repo) because SSM parameter names and S3 keys
|
|
304
|
+
# both accept dots and those paths must stay stable. The title is
|
|
305
|
+
# the only consumer that cannot, so it gets its own sanitised form
|
|
306
|
+
# rather than reshaping the slug everywhere.
|
|
307
|
+
#
|
|
308
|
+
# This broke when CI switched to emitting fully-qualified repo names
|
|
309
|
+
# (github.com/owner/repo) so the attribution keys would match
|
|
310
|
+
# codeburn's. A bare owner/repo slug had no dot and validated fine.
|
|
311
|
+
#
|
|
312
|
+
# printf, not echo: tr -c would otherwise rewrite the trailing
|
|
313
|
+
# newline into a hyphen and leave every title ending in one.
|
|
314
|
+
CR_TITLE=$(printf '%s-ci-diff-scan' "${REPO_SLUG}" | tr -c 'A-Za-z0-9_-' '-' | cut -c1-100)
|
|
315
|
+
|
|
301
316
|
# Get service role (SSM or convention fallback)
|
|
302
317
|
SERVICE_ROLE=$(aws ssm get-parameter --name /prism/continuum/service-role-arn --query Parameter.Value --output text 2>/dev/null || echo "")
|
|
303
318
|
if [ -z "$SERVICE_ROLE" ]; then
|
|
@@ -313,11 +328,17 @@ jobs:
|
|
|
313
328
|
# Create the Code Review resource
|
|
314
329
|
CR_OUTPUT=$(aws securityagent create-code-review \
|
|
315
330
|
--agent-space-id "$AGENT_SPACE_ID" \
|
|
316
|
-
--title "$
|
|
331
|
+
--title "$CR_TITLE" \
|
|
317
332
|
--service-role "$SERVICE_ROLE" \
|
|
318
333
|
--assets "{\"sourceCode\":[{\"s3Location\":\"s3://${SCAN_BUCKET}/repo/${REPO_SLUG}/latest.zip\"}]}" \
|
|
319
334
|
--output json 2>&1) || true
|
|
320
|
-
|
|
335
|
+
# `|| echo ""` is load-bearing. CR_OUTPUT captures stderr via 2>&1,
|
|
336
|
+
# so a failed create leaves AWS's error TEXT here, not JSON. jq exits
|
|
337
|
+
# 5 on a parse error, and under bash -eo pipefail that aborts the
|
|
338
|
+
# whole step -- which made the diagnostic below unreachable and
|
|
339
|
+
# surfaced a bare "Process completed with exit code 5" instead of the
|
|
340
|
+
# actual ValidationException.
|
|
341
|
+
CODE_REVIEW_ID=$(echo "$CR_OUTPUT" | jq -r '.codeReviewId // empty' 2>/dev/null || echo "")
|
|
321
342
|
|
|
322
343
|
if [ -z "$CODE_REVIEW_ID" ]; then
|
|
323
344
|
echo "::error::Could not create the Continuum Code Review resource: ${CR_OUTPUT}"
|
|
@@ -294,14 +294,34 @@ jobs:
|
|
|
294
294
|
git archive HEAD --format=zip -o /tmp/repo.zip
|
|
295
295
|
aws s3 cp /tmp/repo.zip "s3://${SCAN_BUCKET}/repo/${REPO_SLUG}/latest.zip" --quiet
|
|
296
296
|
|
|
297
|
+
# Continuum rejects a title containing anything outside
|
|
298
|
+
# [A-Za-z0-9_-]. REPO_SLUG keeps the dot from the host component
|
|
299
|
+
# (github.com-owner-repo) because SSM parameter names and S3 keys
|
|
300
|
+
# both accept dots and those paths must stay stable. The title is
|
|
301
|
+
# the only consumer that cannot, so it gets its own sanitised form.
|
|
302
|
+
#
|
|
303
|
+
# This broke when CI switched to emitting fully-qualified repo names
|
|
304
|
+
# (github.com/owner/repo) so attribution keys would match codeburn's.
|
|
305
|
+
# A bare owner/repo slug had no dot and validated fine.
|
|
306
|
+
#
|
|
307
|
+
# printf, not echo: tr -c would otherwise rewrite the trailing
|
|
308
|
+
# newline into a hyphen and leave every title ending in one.
|
|
309
|
+
CR_TITLE=$(printf '%s-ci-diff-scan' "${REPO_SLUG}" | tr -c 'A-Za-z0-9_-' '-' | cut -c1-100)
|
|
310
|
+
|
|
297
311
|
# Create the CodeReview
|
|
298
312
|
CR_OUTPUT=$(aws securityagent create-code-review \
|
|
299
313
|
--agent-space-id "$AGENT_SPACE_ID" \
|
|
300
|
-
--title "$
|
|
314
|
+
--title "$CR_TITLE" \
|
|
301
315
|
--service-role "$SERVICE_ROLE" \
|
|
302
316
|
--assets "{\"sourceCode\":[{\"s3Location\":\"s3://${SCAN_BUCKET}/repo/${REPO_SLUG}/latest.zip\"}]}" \
|
|
303
317
|
--output json 2>&1) || true
|
|
304
|
-
|
|
318
|
+
# `|| echo ""` is load-bearing. CR_OUTPUT captures stderr via 2>&1,
|
|
319
|
+
# so a failed create leaves AWS's error TEXT here, not JSON. jq exits
|
|
320
|
+
# 5 on a parse error, and under bash -eo pipefail that aborts the
|
|
321
|
+
# whole step -- which made the diagnostic below unreachable and
|
|
322
|
+
# surfaced a bare "Process completed with exit code 5" instead of the
|
|
323
|
+
# actual ValidationException.
|
|
324
|
+
CODE_REVIEW_ID=$(echo "$CR_OUTPUT" | jq -r '.codeReviewId // empty' 2>/dev/null || echo "")
|
|
305
325
|
|
|
306
326
|
if [[ -z "$CODE_REVIEW_ID" ]]; then
|
|
307
327
|
echo " ⚠️ Bootstrap failed:"
|
|
@@ -4,15 +4,20 @@
|
|
|
4
4
|
* Consumes the prism-d1-ai-usage DynamoDB stream (filtered to COMMIT# items
|
|
5
5
|
* under REPO# pk) and publishes commit attribution metrics to CloudWatch.
|
|
6
6
|
*
|
|
7
|
-
* AI-Origin
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* AI-Origin: prefers the verdict the receiver froze on the item at ingest
|
|
8
|
+
* (ai_origin / ai_tool). Falls back to joining commit → usage spans via traceId
|
|
9
|
+
* for legacy items written before write-time resolution existed. Re-deriving
|
|
10
|
+
* unconditionally was a bug: spans TTL at 90 days, commit facts at 365.
|
|
10
11
|
*
|
|
11
12
|
* Metrics published (namespace: PRISM/D1/Velocity):
|
|
12
13
|
* AICommits — count of ai-generated commits (dimensionless + by Tool)
|
|
13
14
|
* HumanCommits — count of human commits (dimensionless only)
|
|
14
15
|
* RevertedAICommits — ai-generated commits that were later reverted
|
|
15
16
|
* CommitsTotal — all tracked commits regardless of origin
|
|
17
|
+
*
|
|
18
|
+
* A commit seeded by CI as `human` and later upgraded to `ai-generated` by
|
|
19
|
+
* codeburn attribution publishes the AI counts and RETRACTS the superseded
|
|
20
|
+
* human counts with a -1 datum, so Sum-aggregated consumers stay correct.
|
|
16
21
|
*/
|
|
17
22
|
|
|
18
23
|
import {
|
|
@@ -52,6 +57,10 @@ interface StreamImage {
|
|
|
52
57
|
was_reverted?: AttributeValue;
|
|
53
58
|
timestamp?: AttributeValue;
|
|
54
59
|
record_type?: AttributeValue;
|
|
60
|
+
/** Origin verdict frozen by the receiver at ingest. Absent on legacy items. */
|
|
61
|
+
ai_origin?: AttributeValue;
|
|
62
|
+
/** Tool frozen alongside ai_origin. Absent on legacy items. */
|
|
63
|
+
ai_tool?: AttributeValue;
|
|
55
64
|
}
|
|
56
65
|
|
|
57
66
|
interface StreamRecord {
|
|
@@ -69,6 +78,34 @@ interface StreamEvent {
|
|
|
69
78
|
|
|
70
79
|
// ---- Helpers ----
|
|
71
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Resolve a commit's AI origin and tool.
|
|
83
|
+
*
|
|
84
|
+
* Prefers the verdict the receiver froze on the item. That verdict was computed
|
|
85
|
+
* while the usage spans were still alive, whereas this re-derivation runs
|
|
86
|
+
* whenever the stream fires: spans TTL at 90 days and commit facts at 365, so
|
|
87
|
+
* re-deriving an aging commit silently reclassifies it as human. The receiver
|
|
88
|
+
* moved to write-time resolution for exactly that reason (see
|
|
89
|
+
* writeCommitAttribution) — this consumer was still re-deriving, so the two
|
|
90
|
+
* disagreed on the same item.
|
|
91
|
+
*
|
|
92
|
+
* The span join remains as a fallback for legacy items that carry no ai_origin.
|
|
93
|
+
*/
|
|
94
|
+
async function resolveOrigin(
|
|
95
|
+
storedOrigin: string,
|
|
96
|
+
storedTool: string,
|
|
97
|
+
user: string,
|
|
98
|
+
traceId: string,
|
|
99
|
+
): Promise<{ tool: string; isAi: boolean }> {
|
|
100
|
+
if (storedOrigin === 'ai-generated') {
|
|
101
|
+
return { tool: storedTool || 'unknown', isAi: true };
|
|
102
|
+
}
|
|
103
|
+
if (storedOrigin === 'human') {
|
|
104
|
+
return { tool: 'none', isAi: false };
|
|
105
|
+
}
|
|
106
|
+
return resolveAiTool(user, traceId);
|
|
107
|
+
}
|
|
108
|
+
|
|
72
109
|
/** Resolve the AI tool for a commit by joining with usage spans via traceId. */
|
|
73
110
|
async function resolveAiTool(user: string, traceId: string): Promise<{ tool: string; isAi: boolean }> {
|
|
74
111
|
if (!user || !traceId) return { tool: 'none', isAi: false };
|
|
@@ -109,19 +146,32 @@ async function resolveAiTool(user: string, traceId: string): Promise<{ tool: str
|
|
|
109
146
|
return { tool: 'none', isAi: false };
|
|
110
147
|
}
|
|
111
148
|
|
|
112
|
-
/**
|
|
113
|
-
|
|
149
|
+
/**
|
|
150
|
+
* Parse a stream record into commit metadata. Only processes COMMIT# items.
|
|
151
|
+
*
|
|
152
|
+
* Exported for the transition guard in scripts/check-commit-transitions.ts —
|
|
153
|
+
* the handler builds its CloudWatch client at module scope, so the transition
|
|
154
|
+
* logic is only reachable in a test through this function.
|
|
155
|
+
*/
|
|
156
|
+
export function parseCommitRecord(record: StreamRecord): {
|
|
114
157
|
user: string;
|
|
115
158
|
traceId: string;
|
|
116
159
|
wasReverted: boolean;
|
|
117
160
|
inMain: boolean;
|
|
118
161
|
timestamp: string;
|
|
162
|
+
/** Origin frozen on the item by the receiver. Empty for legacy items. */
|
|
163
|
+
storedOrigin: string;
|
|
164
|
+
/** Tool frozen alongside storedOrigin. Empty when unknown. */
|
|
165
|
+
storedTool: string;
|
|
119
166
|
/** Which state transitions this event represents. INSERT = 'new'. A MODIFY
|
|
120
167
|
* can flip in_main and/or was_reverted (receiver writes only when one of
|
|
121
168
|
* them changed). */
|
|
122
169
|
isNew: boolean;
|
|
123
170
|
mergedNow: boolean; // in_main flipped false → true
|
|
124
171
|
revertedNow: boolean; // was_reverted flipped false → true
|
|
172
|
+
/** ai_origin flipped human → ai-generated (codeburn attribution arrived
|
|
173
|
+
* after CI already seeded the commit). */
|
|
174
|
+
originUpgraded: boolean;
|
|
125
175
|
} | null {
|
|
126
176
|
if (record.eventName !== 'INSERT' && record.eventName !== 'MODIFY') return null;
|
|
127
177
|
|
|
@@ -133,18 +183,34 @@ function parseCommitRecord(record: StreamRecord): {
|
|
|
133
183
|
|
|
134
184
|
const inMain = newImage.in_main?.BOOL ?? false;
|
|
135
185
|
const wasReverted = newImage.was_reverted?.BOOL ?? false;
|
|
186
|
+
const storedOrigin = newImage.ai_origin?.S ?? '';
|
|
187
|
+
const storedTool = newImage.ai_tool?.S ?? '';
|
|
136
188
|
|
|
137
|
-
|
|
189
|
+
const isNew = record.eventName === 'INSERT';
|
|
138
190
|
let mergedNow = false;
|
|
139
191
|
let revertedNow = false;
|
|
192
|
+
let originUpgraded = false;
|
|
140
193
|
|
|
141
194
|
if (record.eventName === 'MODIFY') {
|
|
142
195
|
const oldImage = record.dynamodb?.OldImage;
|
|
143
196
|
const oldInMain = oldImage?.in_main?.BOOL ?? false;
|
|
144
197
|
const oldReverted = oldImage?.was_reverted?.BOOL ?? false;
|
|
198
|
+
const oldOrigin = oldImage?.ai_origin?.S ?? '';
|
|
145
199
|
mergedNow = inMain && !oldInMain;
|
|
146
200
|
revertedNow = wasReverted && !oldReverted;
|
|
147
|
-
|
|
201
|
+
// The human → ai-generated upgrade. metrics-processor seeds CI-observed
|
|
202
|
+
// commits as `human` because it cannot know origin; codeburn attribution
|
|
203
|
+
// arrives later and the receiver upgrades them (its ConditionExpression
|
|
204
|
+
// permits exactly `ai_origin = :human AND :ao = :ai`).
|
|
205
|
+
//
|
|
206
|
+
// This used to fall into the `return null` below, because the receiver's
|
|
207
|
+
// Write 1 sets in_main/was_reverted with if_not_exists and therefore
|
|
208
|
+
// changes neither. So the upgrade reached DynamoDB but never CloudWatch:
|
|
209
|
+
// AICommits stayed unemitted and MergedAICommits stayed at zero forever,
|
|
210
|
+
// which blanks the "AI Defect Trend (reverted / merged)" widget since its
|
|
211
|
+
// MathExpression is guarded on `mergedAi2 > 0`.
|
|
212
|
+
originUpgraded = oldOrigin === 'human' && storedOrigin === 'ai-generated';
|
|
213
|
+
if (!mergedNow && !revertedNow && !originUpgraded) return null; // no meaningful transition
|
|
148
214
|
}
|
|
149
215
|
|
|
150
216
|
return {
|
|
@@ -153,9 +219,12 @@ function parseCommitRecord(record: StreamRecord): {
|
|
|
153
219
|
wasReverted,
|
|
154
220
|
inMain,
|
|
155
221
|
timestamp: newImage.timestamp?.S ?? new Date().toISOString(),
|
|
222
|
+
storedOrigin,
|
|
223
|
+
storedTool,
|
|
156
224
|
isNew,
|
|
157
225
|
mergedNow,
|
|
158
226
|
revertedNow,
|
|
227
|
+
originUpgraded,
|
|
159
228
|
};
|
|
160
229
|
}
|
|
161
230
|
|
|
@@ -168,6 +237,7 @@ export async function handler(event: StreamEvent): Promise<void> {
|
|
|
168
237
|
let humanCommits = 0;
|
|
169
238
|
let mergedAi = 0;
|
|
170
239
|
let revertedAi = 0;
|
|
240
|
+
let upgraded = 0;
|
|
171
241
|
|
|
172
242
|
/** Push a Count=1 datum, dimensionless plus an optional Tool-dimensioned twin. */
|
|
173
243
|
const emit = (name: string, ts: Date, tool?: string): void => {
|
|
@@ -177,11 +247,27 @@ export async function handler(event: StreamEvent): Promise<void> {
|
|
|
177
247
|
}
|
|
178
248
|
};
|
|
179
249
|
|
|
250
|
+
/**
|
|
251
|
+
* Retract a previously-published Count by emitting -1.
|
|
252
|
+
*
|
|
253
|
+
* PutMetricData accepts negative values and every consumer of these metrics
|
|
254
|
+
* aggregates with Sum, so a -1 makes the running total correct rather than
|
|
255
|
+
* leaving a permanent over-count. The retraction is emitted at the SAME
|
|
256
|
+
* timestamp basis as the original datum so the pair falls inside or outside
|
|
257
|
+
* any dashboard time range together — retracting at `now` instead would let
|
|
258
|
+
* a range catch the -1 without the +1 and render a negative count.
|
|
259
|
+
*/
|
|
260
|
+
const retract = (name: string, ts: Date): void => {
|
|
261
|
+
datums.push({ MetricName: name, Value: -1, Unit: StandardUnit.Count, Timestamp: ts, Dimensions: [] });
|
|
262
|
+
};
|
|
263
|
+
|
|
180
264
|
for (const record of event.Records ?? []) {
|
|
181
265
|
const commit = parseCommitRecord(record);
|
|
182
266
|
if (!commit) continue;
|
|
183
267
|
|
|
184
|
-
const { tool, isAi } = await
|
|
268
|
+
const { tool, isAi } = await resolveOrigin(
|
|
269
|
+
commit.storedOrigin, commit.storedTool, commit.user, commit.traceId,
|
|
270
|
+
);
|
|
185
271
|
const timestamp = new Date(commit.timestamp);
|
|
186
272
|
// CloudWatch rejects PutMetricData timestamps older than 2 weeks (and one
|
|
187
273
|
// bad member rejects the whole batch, wedging the stream in a retry loop).
|
|
@@ -217,17 +303,46 @@ export async function handler(event: StreamEvent): Promise<void> {
|
|
|
217
303
|
continue;
|
|
218
304
|
}
|
|
219
305
|
|
|
220
|
-
// MODIFY — state transition(s). The receiver writes only when in_main
|
|
221
|
-
// was_reverted actually changed, and a single MODIFY can carry
|
|
222
|
-
|
|
223
|
-
|
|
306
|
+
// MODIFY — state transition(s). The receiver writes only when in_main,
|
|
307
|
+
// was_reverted or ai_origin actually changed, and a single MODIFY can carry
|
|
308
|
+
// more than one.
|
|
309
|
+
|
|
310
|
+
// human → ai-generated. The INSERT already published this commit as human,
|
|
311
|
+
// so the AI counts are published AND the superseded human counts retracted.
|
|
312
|
+
// Without the retraction HumanCommits keeps a phantom that no later event
|
|
313
|
+
// clears, and AI share reads low forever.
|
|
314
|
+
if (commit.originUpgraded) {
|
|
315
|
+
upgraded++;
|
|
316
|
+
aiCommits++;
|
|
317
|
+
retract('HumanCommits', metricTimestamp);
|
|
318
|
+
emit('AICommits', metricTimestamp, tool);
|
|
319
|
+
if (commit.inMain) {
|
|
320
|
+
// in_main was already true at INSERT (CI seeds merged commits), so
|
|
321
|
+
// mergedNow is false and this is the only chance to move the merged
|
|
322
|
+
// count onto the AI series — the one the AI Defect Trend widget reads.
|
|
224
323
|
mergedAi++;
|
|
324
|
+
retract('MergedHumanCommits', metricTimestamp);
|
|
225
325
|
emit('MergedAICommits', metricTimestamp, tool);
|
|
326
|
+
}
|
|
327
|
+
if (commit.wasReverted) {
|
|
328
|
+
revertedAi++;
|
|
329
|
+
emit('RevertedAICommits', metricTimestamp, tool);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
if (commit.mergedNow) {
|
|
334
|
+
if (isAi) {
|
|
335
|
+
// Already counted by the originUpgraded branch above when both fired in
|
|
336
|
+
// the same event; counting twice would double the denominator.
|
|
337
|
+
if (!commit.originUpgraded) {
|
|
338
|
+
mergedAi++;
|
|
339
|
+
emit('MergedAICommits', metricTimestamp, tool);
|
|
340
|
+
}
|
|
226
341
|
} else {
|
|
227
342
|
emit('MergedHumanCommits', metricTimestamp);
|
|
228
343
|
}
|
|
229
344
|
}
|
|
230
|
-
if (commit.revertedNow && isAi) {
|
|
345
|
+
if (commit.revertedNow && isAi && !commit.originUpgraded) {
|
|
231
346
|
revertedAi++;
|
|
232
347
|
emit('RevertedAICommits', metricTimestamp, tool);
|
|
233
348
|
}
|
|
@@ -243,6 +358,7 @@ export async function handler(event: StreamEvent): Promise<void> {
|
|
|
243
358
|
|
|
244
359
|
console.log(
|
|
245
360
|
`[attribution-metrics-publisher] records=${event.Records?.length ?? 0} ` +
|
|
246
|
-
`ai=${aiCommits} human=${humanCommits} merged=${mergedAi} reverted=${revertedAi}
|
|
361
|
+
`ai=${aiCommits} human=${humanCommits} merged=${mergedAi} reverted=${revertedAi} ` +
|
|
362
|
+
`upgraded=${upgraded} datums=${datums.length}`,
|
|
247
363
|
);
|
|
248
364
|
}
|
|
@@ -2,6 +2,7 @@ import { PERSISTED_SECTIONS, validateEventShape } from './event-schema';
|
|
|
2
2
|
import {
|
|
3
3
|
DynamoDBClient,
|
|
4
4
|
PutItemCommand,
|
|
5
|
+
UpdateItemCommand,
|
|
5
6
|
} from '@aws-sdk/client-dynamodb';
|
|
6
7
|
import {
|
|
7
8
|
CloudWatchClient,
|
|
@@ -864,11 +865,25 @@ function mapUnit(unit: string): StandardUnit {
|
|
|
864
865
|
*
|
|
865
866
|
* OR (ai_origin = :human AND :ao = :ai)
|
|
866
867
|
*
|
|
867
|
-
*
|
|
868
|
-
*
|
|
869
|
-
*
|
|
870
|
-
*
|
|
871
|
-
*
|
|
868
|
+
* Two writes, because the two fields have different ownership:
|
|
869
|
+
*
|
|
870
|
+
* ai_origin — either side may write it; the receiver upgrades human→ai and
|
|
871
|
+
* refuses ai→human, so ordering is genuinely irrelevant.
|
|
872
|
+
* in_main — ONLY this function can write it. A PR merge is the only event
|
|
873
|
+
* that knows a commit reached the default branch; the receiver leaves
|
|
874
|
+
* in_main alone on existing items because codeburn cannot know about
|
|
875
|
+
* merges.
|
|
876
|
+
*
|
|
877
|
+
* Write 1 is create-if-absent. Write 2 promotes in_main on an item that
|
|
878
|
+
* already existed, which Write 1 cannot do — its condition is
|
|
879
|
+
* attribute_not_exists(pk), all-or-nothing on the whole item, so a
|
|
880
|
+
* pre-existing commit loses the in_main=true silently.
|
|
881
|
+
*
|
|
882
|
+
* That mattered only for the coding agent. Human codeburn syncs run on a 12h
|
|
883
|
+
* schedule so the merge event effectively always arrives first and Write 1
|
|
884
|
+
* covers it; the agent's emit_commit fires synchronously at commit time, tens
|
|
885
|
+
* of seconds ahead of the merge event, so its commits always pre-existed and
|
|
886
|
+
* reported 0 shipped.
|
|
872
887
|
*
|
|
873
888
|
* The repo format from the CI workflow is bare (owner/name), but codeburn
|
|
874
889
|
* stores with a host prefix (github.com/owner/name). We write under the bare
|
|
@@ -894,6 +909,7 @@ async function seedCommitAttribution(detailType: string, detail: any): Promise<v
|
|
|
894
909
|
const ttl = Math.floor(Date.now() / 1000) + 365 * 86400; // 1 year
|
|
895
910
|
|
|
896
911
|
let seeded = 0;
|
|
912
|
+
let promoted = 0;
|
|
897
913
|
for (let i = 0; i < shas.length; i++) {
|
|
898
914
|
const sha = shas[i];
|
|
899
915
|
const author = authors[i] ?? ''; // parallel array; may be shorter or absent
|
|
@@ -921,10 +937,9 @@ async function seedCommitAttribution(detailType: string, detail: any): Promise<v
|
|
|
921
937
|
gsi_user_sk: { S: `COMMIT#${timestamp}` },
|
|
922
938
|
} : {}),
|
|
923
939
|
},
|
|
924
|
-
//
|
|
925
|
-
// pushed
|
|
926
|
-
//
|
|
927
|
-
// (retry), this is idempotent.
|
|
940
|
+
// Write 1 of 2: create the item only if it does not already exist, so a
|
|
941
|
+
// commit codeburn pushed first keeps its ai_origin=ai-generated verdict
|
|
942
|
+
// (never downgrade ai→human). Idempotent on processor retry.
|
|
928
943
|
ConditionExpression: 'attribute_not_exists(pk)',
|
|
929
944
|
}));
|
|
930
945
|
seeded++;
|
|
@@ -936,9 +951,61 @@ async function seedCommitAttribution(detailType: string, detail: any): Promise<v
|
|
|
936
951
|
throw err;
|
|
937
952
|
}
|
|
938
953
|
}
|
|
954
|
+
|
|
955
|
+
// Write 2 of 2: promote in_main on an item that already existed.
|
|
956
|
+
//
|
|
957
|
+
// in_main has exactly ONE writer: this CI seed. A PR merge is the only
|
|
958
|
+
// event that knows a commit reached the default branch — the receiver
|
|
959
|
+
// deliberately leaves in_main alone on existing items precisely because
|
|
960
|
+
// codeburn cannot know about merges.
|
|
961
|
+
//
|
|
962
|
+
// Write 1 therefore cannot be the only write. Its condition is
|
|
963
|
+
// attribute_not_exists(pk), which is all-or-nothing on the WHOLE item, so
|
|
964
|
+
// when the commit already exists the in_main=true is discarded along with
|
|
965
|
+
// everything else and the commit never counts as shipped.
|
|
966
|
+
//
|
|
967
|
+
// That was invisible for human developers, whose codeburn sync runs on a
|
|
968
|
+
// 12h schedule: the merge event virtually always arrives first, Write 1
|
|
969
|
+
// creates the item, and in_main is correct. The coding agent breaks the
|
|
970
|
+
// assumption — emit_commit fires synchronously at commit time, tens of
|
|
971
|
+
// seconds BEFORE the merge event, so the agent's own commits always lost
|
|
972
|
+
// the race and reported 0 shipped forever.
|
|
973
|
+
//
|
|
974
|
+
// Only ever false→true: CI reports merges, never un-merges, so this is
|
|
975
|
+
// monotonic and cannot fight the receiver. ai_origin/ai_tool are
|
|
976
|
+
// untouched, leaving the frozen attribution verdict authoritative.
|
|
977
|
+
try {
|
|
978
|
+
await dynamoClient.send(new UpdateItemCommand({
|
|
979
|
+
TableName: AI_USAGE_TABLE,
|
|
980
|
+
Key: {
|
|
981
|
+
pk: { S: `REPO#${repo}` },
|
|
982
|
+
sk: { S: `COMMIT#${sha}` },
|
|
983
|
+
},
|
|
984
|
+
UpdateExpression: 'SET in_main = :true, updated_at = :now',
|
|
985
|
+
// Skip the no-op write when in_main is already true, so the DynamoDB
|
|
986
|
+
// stream does not emit a MODIFY the attribution publisher would have
|
|
987
|
+
// to filter out.
|
|
988
|
+
ConditionExpression: 'attribute_exists(pk) AND in_main = :false',
|
|
989
|
+
ExpressionAttributeValues: {
|
|
990
|
+
':true': { BOOL: true },
|
|
991
|
+
':false': { BOOL: false },
|
|
992
|
+
':now': { S: new Date().toISOString() },
|
|
993
|
+
},
|
|
994
|
+
}));
|
|
995
|
+
promoted++;
|
|
996
|
+
} catch (err: any) {
|
|
997
|
+
// Already true, or the item does not exist because Write 1 just created
|
|
998
|
+
// it with in_main=true. Both are the expected steady state.
|
|
999
|
+
if (err.name !== 'ConditionalCheckFailedException') {
|
|
1000
|
+
console.error(`[seedCommitAttribution] Failed to promote in_main for ${sha}:`, err);
|
|
1001
|
+
throw err;
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
939
1004
|
}
|
|
940
1005
|
|
|
941
|
-
if (seeded > 0) {
|
|
942
|
-
console.log(
|
|
1006
|
+
if (seeded > 0 || promoted > 0) {
|
|
1007
|
+
console.log(
|
|
1008
|
+
`[seedCommitAttribution] repo=${repo} seeded=${seeded} promoted_in_main=${promoted} of ${shas.length}`,
|
|
1009
|
+
);
|
|
943
1010
|
}
|
|
944
1011
|
}
|
|
@@ -231,10 +231,33 @@ export function parseOtlpSpans(payload: OtlpPayload): {
|
|
|
231
231
|
spans: ParsedSpan[];
|
|
232
232
|
attributionSpans: ParsedAttributionSpan[];
|
|
233
233
|
rejected: number;
|
|
234
|
+
/**
|
|
235
|
+
* Per-reason rejection counts.
|
|
236
|
+
*
|
|
237
|
+
* `rejected` alone cost an afternoon of code reading to explain a workshop
|
|
238
|
+
* account showing 5 human / 0 AI commits: the receiver knew exactly which
|
|
239
|
+
* branch fired and reported the string 'Spans missing required attributes or
|
|
240
|
+
* failed validation' for all of them. A wire-contract break is now one log
|
|
241
|
+
* line instead of a bisect over published npm tarballs.
|
|
242
|
+
*/
|
|
243
|
+
rejectedReasons: {
|
|
244
|
+
malformedSpan: number;
|
|
245
|
+
noSession: number;
|
|
246
|
+
noSha: number;
|
|
247
|
+
noRepo: number;
|
|
248
|
+
noProvider: number;
|
|
249
|
+
badTokens: number;
|
|
250
|
+
};
|
|
234
251
|
} {
|
|
235
252
|
const spans: ParsedSpan[] = [];
|
|
236
253
|
const attributionSpans: ParsedAttributionSpan[] = [];
|
|
237
254
|
let rejected = 0;
|
|
255
|
+
let rejectedMalformed = 0;
|
|
256
|
+
let rejectedNoSession = 0;
|
|
257
|
+
let rejectedNoSha = 0;
|
|
258
|
+
let rejectedNoRepo = 0;
|
|
259
|
+
let rejectedNoProvider = 0;
|
|
260
|
+
let rejectedBadTokens = 0;
|
|
238
261
|
|
|
239
262
|
for (const rs of payload.resourceSpans ?? []) {
|
|
240
263
|
const resourceAttrs = attrMap(rs.resource?.attributes);
|
|
@@ -249,6 +272,7 @@ export function parseOtlpSpans(payload: OtlpPayload): {
|
|
|
249
272
|
const timestamp = nanoToIso(span.startTimeUnixNano);
|
|
250
273
|
|
|
251
274
|
if (!/^[0-9a-f]{16}$/i.test(spanId) || !timestamp) {
|
|
275
|
+
rejectedMalformed++;
|
|
252
276
|
rejected++;
|
|
253
277
|
continue;
|
|
254
278
|
}
|
|
@@ -256,15 +280,33 @@ export function parseOtlpSpans(payload: OtlpPayload): {
|
|
|
256
280
|
// --- Attribution spans ---
|
|
257
281
|
const attrKind = ATTRIBUTION_SPAN_NAMES[spanName];
|
|
258
282
|
if (attrKind) {
|
|
259
|
-
|
|
260
|
-
|
|
283
|
+
// Session identity moved twice across codeburn releases: `ai.session_id`
|
|
284
|
+
// (<= 0.9.20) -> dropped entirely (0.9.21, 0.9.22) -> `ai.work_unit_id`
|
|
285
|
+
// (0.9.23+). This used to hard-reject when `ai.session_id` was absent,
|
|
286
|
+
// which silently discarded EVERY attribution fact from any codeburn
|
|
287
|
+
// newer than 0.9.20 -- both kinds, because the check ran before the
|
|
288
|
+
// session/commit split. The dashboard then showed 100% human commits
|
|
289
|
+
// with no error surfaced anywhere.
|
|
290
|
+
//
|
|
291
|
+
// Falling back to traceId is sound, not a workaround: codeburn derives
|
|
292
|
+
// the traceId FROM the session id (deriveTraceId(sessionId)), and
|
|
293
|
+
// traceId -- not sessionId -- is the key every join in this file uses
|
|
294
|
+
// (buildUserTraceMap, resolveStoredOrigin). Rejecting on a field the
|
|
295
|
+
// join never reads was strictly worse than accepting the span.
|
|
296
|
+
const sessionId =
|
|
297
|
+
str(attrs.get('ai.session_id')) ||
|
|
298
|
+
str(attrs.get('ai.work_unit_id')) ||
|
|
299
|
+
traceId.toLowerCase();
|
|
300
|
+
if (!sessionId) { rejectedNoSession++; rejected++; continue; }
|
|
261
301
|
|
|
262
302
|
const endTimestamp = nanoToIso(span.endTimeUnixNano) || timestamp;
|
|
263
303
|
const repo = str(attrs.get('git.repo')) || null;
|
|
264
304
|
|
|
265
305
|
if (attrKind === 'commit') {
|
|
266
306
|
const sha = str(attrs.get('git.sha'));
|
|
267
|
-
|
|
307
|
+
// commits without repo can't be joined
|
|
308
|
+
if (!sha) { rejectedNoSha++; rejected++; continue; }
|
|
309
|
+
if (!repo) { rejectedNoRepo++; rejected++; continue; }
|
|
268
310
|
attributionSpans.push({
|
|
269
311
|
kind: 'commit',
|
|
270
312
|
spanId: spanId.toLowerCase(),
|
|
@@ -315,6 +357,7 @@ export function parseOtlpSpans(payload: OtlpPayload): {
|
|
|
315
357
|
// --- Usage spans (existing logic) ---
|
|
316
358
|
const provider = str(attrs.get('ai.provider'));
|
|
317
359
|
if (!provider) {
|
|
360
|
+
rejectedNoProvider++;
|
|
318
361
|
rejected++;
|
|
319
362
|
continue;
|
|
320
363
|
}
|
|
@@ -326,6 +369,7 @@ export function parseOtlpSpans(payload: OtlpPayload): {
|
|
|
326
369
|
inputTokens < 0 || outputTokens < 0 || costUsd < 0 ||
|
|
327
370
|
inputTokens > 100_000_000 || outputTokens > 100_000_000 || costUsd > 10_000
|
|
328
371
|
) {
|
|
372
|
+
rejectedBadTokens++;
|
|
329
373
|
rejected++;
|
|
330
374
|
continue;
|
|
331
375
|
}
|
|
@@ -352,7 +396,19 @@ export function parseOtlpSpans(payload: OtlpPayload): {
|
|
|
352
396
|
}
|
|
353
397
|
}
|
|
354
398
|
|
|
355
|
-
return {
|
|
399
|
+
return {
|
|
400
|
+
spans,
|
|
401
|
+
attributionSpans,
|
|
402
|
+
rejected,
|
|
403
|
+
rejectedReasons: {
|
|
404
|
+
malformedSpan: rejectedMalformed,
|
|
405
|
+
noSession: rejectedNoSession,
|
|
406
|
+
noSha: rejectedNoSha,
|
|
407
|
+
noRepo: rejectedNoRepo,
|
|
408
|
+
noProvider: rejectedNoProvider,
|
|
409
|
+
badTokens: rejectedBadTokens,
|
|
410
|
+
},
|
|
411
|
+
};
|
|
356
412
|
}
|
|
357
413
|
|
|
358
414
|
// ---- Route: discovery doc ----
|
|
@@ -727,7 +783,7 @@ async function handleTraces(event: HttpApiEvent): Promise<HttpApiResponse> {
|
|
|
727
783
|
return jsonResponse(400, { message: 'Body is not valid JSON' });
|
|
728
784
|
}
|
|
729
785
|
|
|
730
|
-
const { spans, attributionSpans, rejected } = parseOtlpSpans(payload);
|
|
786
|
+
const { spans, attributionSpans, rejected, rejectedReasons } = parseOtlpSpans(payload);
|
|
731
787
|
const totalAccepted = spans.length + attributionSpans.length;
|
|
732
788
|
|
|
733
789
|
if (totalAccepted > MAX_BATCH_SIZE) {
|
|
@@ -743,16 +799,30 @@ async function handleTraces(event: HttpApiEvent): Promise<HttpApiResponse> {
|
|
|
743
799
|
// Process attribution spans (new flow)
|
|
744
800
|
const attrWritten = await processAttributionSpans(user, attributionSpans);
|
|
745
801
|
|
|
802
|
+
// Only name the reasons that actually fired — a always-present zero-filled
|
|
803
|
+
// breakdown is noise on the overwhelmingly common clean batch.
|
|
804
|
+
const reasonDetail = Object.entries(rejectedReasons)
|
|
805
|
+
.filter(([, n]) => n > 0)
|
|
806
|
+
.map(([reason, n]) => `${reason}=${n}`)
|
|
807
|
+
.join(' ');
|
|
808
|
+
|
|
746
809
|
console.log(
|
|
747
810
|
`[otel-receiver] user=${user} ` +
|
|
748
811
|
`usage: received=${spans.length} new=${usageWritten} dupes=${spans.length - usageWritten} | ` +
|
|
749
812
|
`attribution: received=${attributionSpans.length} written=${attrWritten} | ` +
|
|
750
|
-
`rejected=${rejected}`,
|
|
813
|
+
`rejected=${rejected}${reasonDetail ? ` (${reasonDetail})` : ''}`,
|
|
751
814
|
);
|
|
752
815
|
|
|
753
816
|
if (rejected > 0) {
|
|
754
817
|
return jsonResponse(200, {
|
|
755
|
-
partialSuccess: {
|
|
818
|
+
partialSuccess: {
|
|
819
|
+
rejectedSpans: rejected,
|
|
820
|
+
// Name the failing validation instead of a generic string: the client
|
|
821
|
+
// prints this, so a contract break is actionable from the CLI output
|
|
822
|
+
// rather than requiring receiver log access.
|
|
823
|
+
errorMessage: `Spans failed validation: ${reasonDetail}`,
|
|
824
|
+
rejectedReasons,
|
|
825
|
+
},
|
|
756
826
|
});
|
|
757
827
|
}
|
|
758
828
|
return jsonResponse(200, {});
|
|
@@ -9,7 +9,9 @@
|
|
|
9
9
|
"synth": "cdk synth",
|
|
10
10
|
"deploy": "cdk deploy --all",
|
|
11
11
|
"diff": "cdk diff",
|
|
12
|
-
"check:metrics": "tsx scripts/check-metric-coverage.ts"
|
|
12
|
+
"check:metrics": "tsx scripts/check-metric-coverage.ts",
|
|
13
|
+
"check:attribution": "tsx scripts/check-attribution-contract.ts",
|
|
14
|
+
"check:transitions": "tsx scripts/check-commit-transitions.ts"
|
|
13
15
|
},
|
|
14
16
|
"dependencies": {
|
|
15
17
|
"aws-cdk-lib": "^2.170.0",
|
|
@@ -17,9 +19,13 @@
|
|
|
17
19
|
"constructs": "^10.4.2"
|
|
18
20
|
},
|
|
19
21
|
"devDependencies": {
|
|
22
|
+
"@aws-sdk/client-cloudwatch": "^3.1121.0",
|
|
23
|
+
"@aws-sdk/client-dynamodb": "^3.1121.0",
|
|
24
|
+
"@aws-sdk/client-s3": "^3.1121.0",
|
|
20
25
|
"@types/node": "^20.17.0",
|
|
26
|
+
"aws-cdk": "^2.170.0",
|
|
21
27
|
"ts-node": "^10.9.2",
|
|
22
|
-
"
|
|
23
|
-
"
|
|
28
|
+
"tsx": "^4.23.12",
|
|
29
|
+
"typescript": "~5.6.0"
|
|
24
30
|
}
|
|
25
31
|
}
|