@prism-d1/cli 1.0.84 → 1.0.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/assets/coding-agent/README.md +824 -0
  2. package/dist/assets/coding-agent/agent.py +381 -0
  3. package/dist/assets/coding-agent/config.py +338 -0
  4. package/dist/assets/coding-agent/deploy/Dockerfile +86 -0
  5. package/dist/assets/coding-agent/deploy/create_harness.py +144 -0
  6. package/dist/assets/coding-agent/deploy/deploy-harness.sh +201 -0
  7. package/dist/assets/coding-agent/deploy/prism-coding-agent.yml +399 -0
  8. package/dist/assets/coding-agent/eval/run_eval.py +430 -0
  9. package/dist/assets/coding-agent/eval/run_harness_eval.py +262 -0
  10. package/dist/assets/coding-agent/eval/stubs/001-tags-element-validation.json +11 -0
  11. package/dist/assets/coding-agent/eval/stubs/002-status-filter.json +11 -0
  12. package/dist/assets/coding-agent/eval/stubs/003-refuse-test-deletion.json +11 -0
  13. package/dist/assets/coding-agent/fixtures/001-tags-element-validation.json +12 -0
  14. package/dist/assets/coding-agent/fixtures/002-status-filter.json +12 -0
  15. package/dist/assets/coding-agent/fixtures/003-refuse-test-deletion.json +10 -0
  16. package/dist/assets/coding-agent/pyproject.toml +28 -0
  17. package/dist/assets/coding-agent/requirements.txt +8 -0
  18. package/dist/assets/coding-agent/system_prompt.py +290 -0
  19. package/dist/assets/coding-agent/tools/__init__.py +11 -0
  20. package/dist/assets/coding-agent/tools/create_pr.py +141 -0
  21. package/dist/assets/coding-agent/tools/git_ops.py +197 -0
  22. package/dist/assets/github-workflows/prism-agent-eval.yml +2 -1
  23. package/dist/assets/github-workflows/prism-ai-metrics.yml +12 -3
  24. package/dist/assets/github-workflows/prism-eval-gate-kiro.yml +4 -3
  25. package/dist/assets/github-workflows/prism-eval-gate.yml +3 -2
  26. package/dist/assets/gitlab-workflows/prism-agent-eval.yml +2 -1
  27. package/dist/assets/gitlab-workflows/prism-ai-metrics.yml +3 -2
  28. package/dist/assets/gitlab-workflows/prism-eval-gate.yml +2 -1
  29. package/dist/assets/infra/lib/constructs/otel-collector-construct.ts +134 -2
  30. package/dist/assets/infra/lib/dashboard-stack.ts +79 -95
  31. package/dist/assets/infra/lib/lambda/metrics-processor.ts +99 -1
  32. package/dist/assets/infra/lib/lambda/otel-receiver.ts +318 -29
  33. package/dist/assets/infra/lib/lambda/productivity-widget.ts +85 -4
  34. package/dist/assets/infra/lib/lambda/velocity-widget.ts +265 -0
  35. package/dist/assets/infra/lib/metrics-pipeline-stack.ts +5 -0
  36. package/dist/src/commands/bootstrapper/install-coding-agent.d.ts +62 -0
  37. package/dist/src/commands/bootstrapper/install-coding-agent.d.ts.map +1 -0
  38. package/dist/src/commands/bootstrapper/install-coding-agent.js +614 -0
  39. package/dist/src/commands/bootstrapper/install-coding-agent.js.map +1 -0
  40. package/dist/src/commands/bootstrapper/setup-github-oidc.d.ts.map +1 -1
  41. package/dist/src/commands/bootstrapper/setup-github-oidc.js +23 -0
  42. package/dist/src/commands/bootstrapper/setup-github-oidc.js.map +1 -1
  43. package/package.json +2 -2
@@ -92,47 +92,19 @@ export class DashboardStack extends cdk.Stack {
92
92
  });
93
93
 
94
94
  teamDashboard.addWidgets(
95
- new cloudwatch.GraphWidget({
96
- title: 'Commits / Day (AI vs Human)',
97
- left: [
98
- dailySum('AICommits', 'AI commits'),
99
- dailySum('HumanCommits', 'Human commits'),
100
- ],
101
- view: cloudwatch.GraphWidgetView.BAR,
102
- stacked: true,
103
- width: 8,
104
- height: 6,
105
- leftYAxis: { min: 0, label: 'Commits' },
106
- }),
107
- new cloudwatch.GraphWidget({
108
- title: 'Merge Ratio: AI vs Human',
109
- left: [
110
- // Both lines from attribution spans. Divergence is the signal:
111
- // AI code merging at a materially lower rate than human code means
112
- // review friction or quality problems, not just "AI is used a lot".
113
- new cloudwatch.MathExpression({
114
- expression: 'FILL(IF(FILL(aiCommits, 0) > 0, 100 * FILL(mergedAi, 0) / FILL(aiCommits, 0)), REPEAT)',
115
- usingMetrics: {
116
- mergedAi: dailySum('MergedAICommits'),
117
- aiCommits: dailySum('AICommits'),
118
- },
119
- label: 'AI merge rate (%)',
120
- period: cdk.Duration.days(1),
121
- }),
122
- new cloudwatch.MathExpression({
123
- expression: 'FILL(IF(FILL(humanCommits, 0) > 0, 100 * FILL(mergedHuman, 0) / FILL(humanCommits, 0)), REPEAT)',
124
- usingMetrics: {
125
- mergedHuman: dailySum('MergedHumanCommits'),
126
- humanCommits: dailySum('HumanCommits'),
127
- },
128
- label: 'Human merge rate (%)',
129
- period: cdk.Duration.days(1),
130
- }),
131
- ],
132
- width: 8,
95
+ // Commits / Day and Merge Ratio replaced by DDB-backed custom widget to
96
+ // avoid CloudWatch increment-only double-count when CI-seeded human items
97
+ // are later upgraded to AI by codeburn attribution.
98
+ ...(props?.velocityWidgetArn ? [new cloudwatch.CustomWidget({
99
+ functionArn: props.velocityWidgetArn,
100
+ title: 'Commits & Merge Rate (AI vs Human)',
101
+ width: 16,
133
102
  height: 6,
134
- leftYAxis: { min: 0, max: 100, label: 'Percent' },
135
- }),
103
+ params: { view: 'commits-trend' },
104
+ updateOnRefresh: true,
105
+ updateOnResize: false,
106
+ updateOnTimeRangeChange: true,
107
+ })] : []),
136
108
  new cloudwatch.GraphWidget({
137
109
  title: 'AI Defect Trend (reverted / merged)',
138
110
  left: [
@@ -232,18 +204,17 @@ export class DashboardStack extends cdk.Stack {
232
204
  });
233
205
 
234
206
  execDashboard.addWidgets(
235
- new cloudwatch.GraphWidget({
236
- title: 'AI vs Human Commits (Weekly)',
237
- left: [
238
- weeklySum('AICommits', 'AI commits'),
239
- weeklySum('HumanCommits', 'Human commits'),
240
- ],
241
- view: cloudwatch.GraphWidgetView.BAR,
242
- stacked: true,
207
+ // AI vs Human Commits (Weekly) replaced by DDB-backed custom widget.
208
+ ...(props?.velocityWidgetArn ? [new cloudwatch.CustomWidget({
209
+ functionArn: props.velocityWidgetArn,
210
+ title: 'AI vs Human Commits & Merge Rate',
243
211
  width: 8,
244
212
  height: 6,
245
- leftYAxis: { min: 0, label: 'Commits' },
246
- }),
213
+ params: { view: 'commits-trend' },
214
+ updateOnRefresh: true,
215
+ updateOnResize: false,
216
+ updateOnTimeRangeChange: true,
217
+ })] : []),
247
218
  new cloudwatch.GraphWidget({
248
219
  title: 'AI Spend Trend (Weekly)',
249
220
  left: [weeklySum('AICostUSD', 'Spend ($)')],
@@ -273,27 +244,11 @@ export class DashboardStack extends cdk.Stack {
273
244
  );
274
245
 
275
246
  // Row 4: quality — "is AI code as reliable as human code?"
247
+ // Merge Rate: AI vs Human is already included in the commits-trend custom
248
+ // widget above (DDB-backed, avoids CloudWatch double-count). Only the AI
249
+ // Defect Trend (RevertedAICommits / MergedAICommits) remains as CloudWatch
250
+ // native because it uses AI-only metrics with no upgrade double-count issue.
276
251
  execDashboard.addWidgets(
277
- new cloudwatch.GraphWidget({
278
- title: 'Merge Rate: AI vs Human (Weekly)',
279
- left: [
280
- new cloudwatch.MathExpression({
281
- expression: 'FILL(IF(FILL(xAi, 0) > 0, 100 * FILL(xMergedAi, 0) / FILL(xAi, 0)), REPEAT)',
282
- usingMetrics: { xMergedAi: weeklySum('MergedAICommits'), xAi: weeklySum('AICommits') },
283
- label: 'AI merge rate (%)',
284
- period: cdk.Duration.days(7),
285
- }),
286
- new cloudwatch.MathExpression({
287
- expression: 'FILL(IF(FILL(xHuman, 0) > 0, 100 * FILL(xMergedHuman, 0) / FILL(xHuman, 0)), REPEAT)',
288
- usingMetrics: { xMergedHuman: weeklySum('MergedHumanCommits'), xHuman: weeklySum('HumanCommits') },
289
- label: 'Human merge rate (%)',
290
- period: cdk.Duration.days(7),
291
- }),
292
- ],
293
- width: 12,
294
- height: 6,
295
- leftYAxis: { min: 0, max: 100, label: 'Percent' },
296
- }),
297
252
  new cloudwatch.GraphWidget({
298
253
  title: 'AI Defect Trend (Weekly)',
299
254
  left: [
@@ -739,11 +694,26 @@ export class DashboardStack extends cdk.Stack {
739
694
  });
740
695
 
741
696
  devDashboard.addWidgets(
742
- ratioKpi('AI Share of Commits (%)', 'IF(FILL(kTotal, 0) > 0, 100 * FILL(kAi, 0) / FILL(kTotal, 0))', {
743
- kAi: dailyMetric('AICommits', 'kAi'), kTotal: dailyMetric('CommitsTotal', 'kTotal'),
697
+ // AI Share and AI Merge Rate are DDB-backed (accurate, no CloudWatch delay)
698
+ new cloudwatch.CustomWidget({
699
+ functionArn: props.velocityWidgetArn ?? '',
700
+ title: 'AI Share of Commits',
701
+ width: 6,
702
+ height: 4,
703
+ params: { view: 'ai-share-kpi' },
704
+ updateOnRefresh: true,
705
+ updateOnResize: false,
706
+ updateOnTimeRangeChange: true,
744
707
  }),
745
- ratioKpi('AI Merge Rate (%)', 'IF(FILL(kAi2, 0) > 0, 100 * FILL(kMerged, 0) / FILL(kAi2, 0))', {
746
- kMerged: dailyMetric('MergedAICommits', 'kMerged'), kAi2: dailyMetric('AICommits', 'kAi2'),
708
+ new cloudwatch.CustomWidget({
709
+ functionArn: props.velocityWidgetArn ?? '',
710
+ title: 'AI Merge Rate',
711
+ width: 6,
712
+ height: 4,
713
+ params: { view: 'merge-rate-kpi' },
714
+ updateOnRefresh: true,
715
+ updateOnResize: false,
716
+ updateOnTimeRangeChange: true,
747
717
  }),
748
718
  ratioKpi('Cost per Shipped Commit ($)', 'IF(FILL(kMerged2, 0) > 0, FILL(kCost, 0) / FILL(kMerged2, 0))', {
749
719
  kCost: dailyMetric('AICostUSD', 'kCost'), kMerged2: dailyMetric('MergedAICommits', 'kMerged2'),
@@ -757,19 +727,18 @@ export class DashboardStack extends cdk.Stack {
757
727
  }),
758
728
  );
759
729
 
760
- // --- Row 2: Org trends (native interactive charts) ---
730
+ // --- Row 2: Org trends ---
761
731
  devDashboard.addWidgets(
762
- new cloudwatch.GraphWidget({
732
+ // Commits bar chart — just AI vs Human stacked bars, no KPIs or merge rate
733
+ new cloudwatch.CustomWidget({
734
+ functionArn: props.velocityWidgetArn ?? '',
763
735
  title: 'Commits / Day (AI vs Human)',
764
- left: [
765
- dailyMetric('AICommits', 'AI commits'),
766
- dailyMetric('HumanCommits', 'Human commits'),
767
- ],
768
- view: cloudwatch.GraphWidgetView.BAR,
769
- stacked: true,
770
736
  width: 8,
771
737
  height: 6,
772
- leftYAxis: { min: 0, label: 'Commits' },
738
+ params: { view: 'commits-bar' },
739
+ updateOnRefresh: true,
740
+ updateOnResize: false,
741
+ updateOnTimeRangeChange: true,
773
742
  }),
774
743
  new cloudwatch.GraphWidget({
775
744
  title: 'AI Spend / Day',
@@ -778,23 +747,38 @@ export class DashboardStack extends cdk.Stack {
778
747
  height: 6,
779
748
  leftYAxis: { min: 0, label: 'USD' },
780
749
  }),
781
- new cloudwatch.GraphWidget({
782
- title: 'AI Merge Ratio Trend',
783
- left: [
784
- new cloudwatch.MathExpression({
785
- expression: 'FILL(IF(FILL(tAi, 0) > 0, 100 * FILL(tMerged, 0) / FILL(tAi, 0)), REPEAT)',
786
- usingMetrics: { tMerged: dailyMetric('MergedAICommits', 'tMerged'), tAi: dailyMetric('AICommits', 'tAi') },
787
- label: 'Merge ratio (%)',
788
- period: cdk.Duration.days(1),
789
- }),
790
- ],
750
+ // Merge Rate Trend — DDB-backed line chart replacing CloudWatch AI Merge Ratio Trend
751
+ new cloudwatch.CustomWidget({
752
+ functionArn: props.velocityWidgetArn ?? '',
753
+ title: 'Merge Rate Trend (AI vs Human)',
791
754
  width: 8,
792
755
  height: 6,
793
- leftYAxis: { min: 0, max: 100, label: 'Percent' },
756
+ params: { view: 'merge-rate-trend' },
757
+ updateOnRefresh: true,
758
+ updateOnResize: false,
759
+ updateOnTimeRangeChange: true,
760
+ }),
761
+ );
762
+
763
+ // --- Row 3: The coding agent's own work ---
764
+ // Its own row rather than a column in the team table: autonomous work is not
765
+ // human-assisted AI use, and the receiver keeps it out of `totals` for the
766
+ // same reason. Placed above the human table because an agent that ships more
767
+ // than a developer should not be discovered by scrolling.
768
+ devDashboard.addWidgets(
769
+ new cloudwatch.CustomWidget({
770
+ functionArn: props.productivityWidgetArn,
771
+ title: 'PRISM Coding Agent — issues, commits and spend',
772
+ width: 24,
773
+ height: 7,
774
+ params: { view: 'coding-agent' },
775
+ updateOnRefresh: true,
776
+ updateOnResize: false,
777
+ updateOnTimeRangeChange: true,
794
778
  }),
795
779
  );
796
780
 
797
- // --- Row 3: Team comparison table (custom widget, view=table) ---
781
+ // --- Row 4: Team comparison table (custom widget, view=table) ---
798
782
  devDashboard.addWidgets(
799
783
  new cloudwatch.CustomWidget({
800
784
  functionArn: props.productivityWidgetArn,
@@ -196,6 +196,11 @@ const cloudwatchClient = new CloudWatchClient({});
196
196
  const EVENTS_TABLE = process.env.EVENTS_TABLE!;
197
197
  const METADATA_TABLE = process.env.METADATA_TABLE!;
198
198
  const METRIC_NAMESPACE = process.env.METRIC_NAMESPACE ?? 'PRISM/D1/Velocity';
199
+ // Optional: when set, PR events seed the attribution store with default-human
200
+ // COMMIT# items so that commits not yet pushed by codeburn are visible as human
201
+ // rather than absent. The otel-receiver's condition expression allows a later
202
+ // codeburn push to upgrade human → ai-generated, so ordering does not matter.
203
+ const AI_USAGE_TABLE = process.env.AI_USAGE_TABLE;
199
204
 
200
205
 
201
206
  // ---- Handler ----
@@ -220,9 +225,10 @@ export async function handler(event: EventBridgeEvent): Promise<void> {
220
225
  writeEventToDynamo(detailType, detail),
221
226
  writeMetadataToDynamo(detailType, detail),
222
227
  publishCloudWatchMetrics(detailType, detail),
228
+ seedCommitAttribution(detailType, detail),
223
229
  ]);
224
230
 
225
- const labels = ['writeEventToDynamo', 'writeMetadataToDynamo', 'publishCloudWatchMetrics'];
231
+ const labels = ['writeEventToDynamo', 'writeMetadataToDynamo', 'publishCloudWatchMetrics', 'seedCommitAttribution'];
226
232
  results.forEach((result, idx) => {
227
233
  if (result.status === 'rejected') {
228
234
  console.error(`[metrics-processor] ${labels[idx]} FAILED:`, result.reason);
@@ -844,3 +850,95 @@ function mapUnit(unit: string): StandardUnit {
844
850
  };
845
851
  return unitMap[unit?.toLowerCase()] ?? StandardUnit.None;
846
852
  }
853
+
854
+
855
+ // ---- Commit attribution seeding ----
856
+
857
+ /**
858
+ * When a prism.d1.pr event arrives with commit_shas, seed a COMMIT# item for
859
+ * each SHA in the attribution store with ai_origin=human and in_main=true.
860
+ *
861
+ * This ensures human commits are explicitly represented rather than invisible.
862
+ * The otel-receiver's condition expression allows a later codeburn push to
863
+ * upgrade human → ai-generated:
864
+ *
865
+ * OR (ai_origin = :human AND :ao = :ai)
866
+ *
867
+ * So ordering between PR merge and codeburn sync does not matter:
868
+ * - If codeburn pushed first: item already has ai_origin=ai-generated, this
869
+ * write is rejected by the condition (never downgrade ai→human). ✓
870
+ * - If PR merges first: item is written as human, codeburn upgrades later. ✓
871
+ * - Simultaneous: DDB conditional write is atomic, one wins. ✓
872
+ *
873
+ * The repo format from the CI workflow is bare (owner/name), but codeburn
874
+ * stores with a host prefix (github.com/owner/name). We write under the bare
875
+ * form because the /v1/repos and /v1/productivity endpoints scan ALL COMMIT#
876
+ * items regardless of repo format, and repoKeyCandidates() in the security
877
+ * processor already handles both forms for point lookups.
878
+ */
879
+ async function seedCommitAttribution(detailType: string, detail: any): Promise<void> {
880
+ // Only process prism.d1.pr events that carry commit SHAs
881
+ if (detailType !== 'prism.d1.pr') return;
882
+ if (!AI_USAGE_TABLE) return;
883
+
884
+ const pr = detail.pr as PrDetail | undefined;
885
+ const shas = pr?.commit_shas;
886
+ if (!shas || shas.length === 0) return;
887
+
888
+ const authors: string[] = (pr as any)?.commit_authors ?? [];
889
+ const repo = detail.repo as string;
890
+ // CI workflows now emit fully-qualified repo names (github.com/owner/repo
891
+ // or gitlab.com/group/project) matching codeburn's convention. No
892
+ // normalization needed — the ConditionExpression hits existing items directly.
893
+ const timestamp = detail.timestamp as string ?? new Date().toISOString();
894
+ const ttl = Math.floor(Date.now() / 1000) + 365 * 86400; // 1 year
895
+
896
+ let seeded = 0;
897
+ for (let i = 0; i < shas.length; i++) {
898
+ const sha = shas[i];
899
+ const author = authors[i] ?? ''; // parallel array; may be shorter or absent
900
+ try {
901
+ await dynamoClient.send(new PutItemCommand({
902
+ TableName: AI_USAGE_TABLE,
903
+ Item: {
904
+ pk: { S: `REPO#${repo}` },
905
+ sk: { S: `COMMIT#${sha}` },
906
+ record_type: { S: 'OTEL_ATTR_COMMIT' },
907
+ ai_origin: { S: 'human' },
908
+ ai_tool: { S: 'none' },
909
+ ai_model: { S: '' },
910
+ origin_source: { S: 'ci-seeded' },
911
+ in_main: { BOOL: true },
912
+ was_reverted: { BOOL: false },
913
+ timestamp: { S: timestamp },
914
+ updated_at: { S: new Date().toISOString() },
915
+ ttl: { N: String(ttl) },
916
+ // Author email from git log --format='%ae'. Enables per-developer
917
+ // attribution in /v1/productivity even before codeburn syncs.
918
+ ...(author ? {
919
+ user: { S: author },
920
+ gsi_user: { S: `USER#${author}` },
921
+ gsi_user_sk: { S: `COMMIT#${timestamp}` },
922
+ } : {}),
923
+ },
924
+ // Only write if the item does NOT already exist. If codeburn already
925
+ // pushed this commit (with ai_origin=ai-generated), do not overwrite.
926
+ // If the metrics-processor already seeded it on a prior invocation
927
+ // (retry), this is idempotent.
928
+ ConditionExpression: 'attribute_not_exists(pk)',
929
+ }));
930
+ seeded++;
931
+ } catch (err: any) {
932
+ // ConditionalCheckFailedException means item already exists — expected
933
+ // when codeburn pushed first or on retry. Not an error.
934
+ if (err.name !== 'ConditionalCheckFailedException') {
935
+ console.error(`[seedCommitAttribution] Failed to seed ${sha}:`, err);
936
+ throw err;
937
+ }
938
+ }
939
+ }
940
+
941
+ if (seeded > 0) {
942
+ console.log(`[seedCommitAttribution] Seeded ${seeded}/${shas.length} commits as human for ${repo}`);
943
+ }
944
+ }