@avocadostudio-ai/orchestrator-core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/LICENSE +201 -0
  2. package/dist/agent/agent-context.d.ts +19 -0
  3. package/dist/agent/agent-context.js +67 -0
  4. package/dist/agent/agent-logger.d.ts +5 -0
  5. package/dist/agent/agent-logger.js +22 -0
  6. package/dist/agent/agent-loop-openai.d.ts +8 -0
  7. package/dist/agent/agent-loop-openai.js +172 -0
  8. package/dist/agent/agent-loop.d.ts +56 -0
  9. package/dist/agent/agent-loop.js +167 -0
  10. package/dist/agent/agent-provider.d.ts +28 -0
  11. package/dist/agent/agent-provider.js +63 -0
  12. package/dist/agent/agent-tools.d.ts +28 -0
  13. package/dist/agent/agent-tools.js +899 -0
  14. package/dist/agent/context/editing-guidelines.md +46 -0
  15. package/dist/agent/context/role.md +39 -0
  16. package/dist/agent/integration-prompt.d.ts +9 -0
  17. package/dist/agent/integration-prompt.js +154 -0
  18. package/dist/agent/sites-agent-context.d.ts +12 -0
  19. package/dist/agent/sites-agent-context.js +316 -0
  20. package/dist/agent/sites-agent-shared.d.ts +161 -0
  21. package/dist/agent/sites-agent-shared.js +1101 -0
  22. package/dist/agent/sites-agent-tools.d.ts +18 -0
  23. package/dist/agent/sites-agent-tools.js +1227 -0
  24. package/dist/chat/anthropic-cache.d.ts +20 -0
  25. package/dist/chat/anthropic-cache.js +54 -0
  26. package/dist/chat/anthropic-planner.d.ts +98 -0
  27. package/dist/chat/anthropic-planner.js +1012 -0
  28. package/dist/chat/changelog-coverage-validator.d.ts +37 -0
  29. package/dist/chat/changelog-coverage-validator.js +215 -0
  30. package/dist/chat/chat-pipeline-context.d.ts +211 -0
  31. package/dist/chat/chat-pipeline-context.js +249 -0
  32. package/dist/chat/chat-pipeline-deterministic.d.ts +61 -0
  33. package/dist/chat/chat-pipeline-deterministic.js +407 -0
  34. package/dist/chat/chat-pipeline-image.d.ts +86 -0
  35. package/dist/chat/chat-pipeline-image.js +897 -0
  36. package/dist/chat/chat-pipeline-shared.d.ts +69 -0
  37. package/dist/chat/chat-pipeline-shared.js +212 -0
  38. package/dist/chat/chat-pipeline-translation.d.ts +27 -0
  39. package/dist/chat/chat-pipeline-translation.js +417 -0
  40. package/dist/chat/chat-pipeline-ui.d.ts +14 -0
  41. package/dist/chat/chat-pipeline-ui.js +244 -0
  42. package/dist/chat/chat-pipeline.d.ts +99 -0
  43. package/dist/chat/chat-pipeline.js +3999 -0
  44. package/dist/chat/decomposer.d.ts +21 -0
  45. package/dist/chat/decomposer.js +65 -0
  46. package/dist/chat/gemini-planner.d.ts +70 -0
  47. package/dist/chat/gemini-planner.js +541 -0
  48. package/dist/chat/hallucination-validator.d.ts +36 -0
  49. package/dist/chat/hallucination-validator.js +110 -0
  50. package/dist/chat/locale-strings.d.ts +47 -0
  51. package/dist/chat/locale-strings.js +100 -0
  52. package/dist/chat/plan-json-schema.d.ts +133 -0
  53. package/dist/chat/plan-json-schema.js +112 -0
  54. package/dist/chat/planner-types.d.ts +120 -0
  55. package/dist/chat/planner-types.js +66 -0
  56. package/dist/chat/planner.d.ts +148 -0
  57. package/dist/chat/planner.js +1361 -0
  58. package/dist/chat/prompts.d.ts +67 -0
  59. package/dist/chat/prompts.js +356 -0
  60. package/dist/chat/provider-routing.d.ts +14 -0
  61. package/dist/chat/provider-routing.js +27 -0
  62. package/dist/chat/variation-pipeline.d.ts +135 -0
  63. package/dist/chat/variation-pipeline.js +837 -0
  64. package/dist/chat/vision-alt-generator.d.ts +35 -0
  65. package/dist/chat/vision-alt-generator.js +152 -0
  66. package/dist/cms/adapter.d.ts +62 -0
  67. package/dist/cms/adapter.js +1 -0
  68. package/dist/cms/bootstrap.d.ts +17 -0
  69. package/dist/cms/bootstrap.js +85 -0
  70. package/dist/cms/editor-api-adapter.d.ts +23 -0
  71. package/dist/cms/editor-api-adapter.js +71 -0
  72. package/dist/cms/index.d.ts +4 -0
  73. package/dist/cms/index.js +3 -0
  74. package/dist/cms/json-file-adapter.d.ts +11 -0
  75. package/dist/cms/json-file-adapter.js +62 -0
  76. package/dist/demo-mode.d.ts +59 -0
  77. package/dist/demo-mode.js +201 -0
  78. package/dist/errors.d.ts +67 -0
  79. package/dist/errors.js +129 -0
  80. package/dist/http/chat-stream-resumable.d.ts +108 -0
  81. package/dist/http/chat-stream-resumable.js +290 -0
  82. package/dist/http/chat-stream.d.ts +99 -0
  83. package/dist/http/chat-stream.js +92 -0
  84. package/dist/image/gdrive-client.d.ts +22 -0
  85. package/dist/image/gdrive-client.js +215 -0
  86. package/dist/image/image-helpers.d.ts +95 -0
  87. package/dist/image/image-helpers.js +488 -0
  88. package/dist/index.d.ts +1 -0
  89. package/dist/index.js +1 -0
  90. package/dist/jira/jira-approval.d.ts +22 -0
  91. package/dist/jira/jira-approval.js +51 -0
  92. package/dist/jira/jira-client.d.ts +44 -0
  93. package/dist/jira/jira-client.js +313 -0
  94. package/dist/jira/jira-poller.d.ts +46 -0
  95. package/dist/jira/jira-poller.js +184 -0
  96. package/dist/jira/jira-processor.d.ts +103 -0
  97. package/dist/jira/jira-processor.js +1085 -0
  98. package/dist/jira/jira-types.d.ts +117 -0
  99. package/dist/jira/jira-types.js +38 -0
  100. package/dist/logger.d.ts +12 -0
  101. package/dist/logger.js +28 -0
  102. package/dist/migration/mcp-server-stdio.d.ts +8 -0
  103. package/dist/migration/mcp-server-stdio.js +672 -0
  104. package/dist/migration/migration-prompt.d.ts +7 -0
  105. package/dist/migration/migration-prompt.js +197 -0
  106. package/dist/migration/migration-tools.d.ts +17 -0
  107. package/dist/migration/migration-tools.js +159 -0
  108. package/dist/migration/scrape-cache.d.ts +9 -0
  109. package/dist/migration/scrape-cache.js +19 -0
  110. package/dist/nlp/deterministic-planner-context.d.ts +141 -0
  111. package/dist/nlp/deterministic-planner-context.js +362 -0
  112. package/dist/nlp/deterministic-planner-pages.d.ts +26 -0
  113. package/dist/nlp/deterministic-planner-pages.js +170 -0
  114. package/dist/nlp/deterministic-planner-patches.d.ts +80 -0
  115. package/dist/nlp/deterministic-planner-patches.js +508 -0
  116. package/dist/nlp/deterministic-planner-refs.d.ts +33 -0
  117. package/dist/nlp/deterministic-planner-refs.js +164 -0
  118. package/dist/nlp/deterministic-planner-suggestions.d.ts +49 -0
  119. package/dist/nlp/deterministic-planner-suggestions.js +579 -0
  120. package/dist/nlp/deterministic-planner.d.ts +85 -0
  121. package/dist/nlp/deterministic-planner.js +1631 -0
  122. package/dist/nlp/intent-detection.d.ts +309 -0
  123. package/dist/nlp/intent-detection.js +730 -0
  124. package/dist/nlp/intent-helpers.d.ts +15 -0
  125. package/dist/nlp/intent-helpers.js +243 -0
  126. package/dist/nlp/intent-patterns.d.ts +40 -0
  127. package/dist/nlp/intent-patterns.js +223 -0
  128. package/dist/nlp/plan-normalizer.d.ts +41 -0
  129. package/dist/nlp/plan-normalizer.js +1537 -0
  130. package/dist/ops/destructive-action-gate.d.ts +44 -0
  131. package/dist/ops/destructive-action-gate.js +90 -0
  132. package/dist/ops/ops-engine.d.ts +151 -0
  133. package/dist/ops/ops-engine.js +1394 -0
  134. package/dist/publish/diff-engine.d.ts +18 -0
  135. package/dist/publish/diff-engine.js +305 -0
  136. package/dist/publish/publish-helpers.d.ts +87 -0
  137. package/dist/publish/publish-helpers.js +521 -0
  138. package/dist/publish/publish-target-registry.d.ts +7 -0
  139. package/dist/publish/publish-target-registry.js +61 -0
  140. package/dist/publish/publish-target.d.ts +81 -0
  141. package/dist/publish/publish-target.js +1 -0
  142. package/dist/publish/targets/deploy-hook.d.ts +13 -0
  143. package/dist/publish/targets/deploy-hook.js +123 -0
  144. package/dist/publish/targets/git.d.ts +13 -0
  145. package/dist/publish/targets/git.js +55 -0
  146. package/dist/publish/targets/site-contract.d.ts +19 -0
  147. package/dist/publish/targets/site-contract.js +124 -0
  148. package/dist/state/content-source.d.ts +17 -0
  149. package/dist/state/content-source.js +1 -0
  150. package/dist/state/in-memory-content-source.d.ts +27 -0
  151. package/dist/state/in-memory-content-source.js +51 -0
  152. package/dist/state/session-lock.d.ts +13 -0
  153. package/dist/state/session-lock.js +29 -0
  154. package/dist/state/session-state.d.ts +310 -0
  155. package/dist/state/session-state.js +1083 -0
  156. package/dist/state/sqlite-store-singleton.d.ts +31 -0
  157. package/dist/state/sqlite-store-singleton.js +170 -0
  158. package/dist/state/sqlite-store.d.ts +135 -0
  159. package/dist/state/sqlite-store.js +421 -0
  160. package/dist/telemetry/chat-telemetry.d.ts +105 -0
  161. package/dist/telemetry/chat-telemetry.js +247 -0
  162. package/dist/telemetry/eval-candidate-store.d.ts +50 -0
  163. package/dist/telemetry/eval-candidate-store.js +120 -0
  164. package/dist/telemetry/feedback-store.d.ts +34 -0
  165. package/dist/telemetry/feedback-store.js +76 -0
  166. package/dist/telemetry/jira-telemetry.d.ts +57 -0
  167. package/dist/telemetry/jira-telemetry.js +68 -0
  168. package/dist/telemetry/migration-telemetry.d.ts +35 -0
  169. package/dist/telemetry/migration-telemetry.js +40 -0
  170. package/dist/telemetry/usage.d.ts +24 -0
  171. package/dist/telemetry/usage.js +80 -0
  172. package/dist/tools/builtin-registrations.d.ts +12 -0
  173. package/dist/tools/builtin-registrations.js +33 -0
  174. package/dist/tools/builtins/gdrive-browse.d.ts +3 -0
  175. package/dist/tools/builtins/gdrive-browse.js +68 -0
  176. package/dist/tools/builtins/image-generate.d.ts +3 -0
  177. package/dist/tools/builtins/image-generate.js +211 -0
  178. package/dist/tools/builtins/unsplash-get-by-id.d.ts +23 -0
  179. package/dist/tools/builtins/unsplash-get-by-id.js +119 -0
  180. package/dist/tools/builtins/unsplash-search.d.ts +3 -0
  181. package/dist/tools/builtins/unsplash-search.js +74 -0
  182. package/dist/tools/executor.d.ts +23 -0
  183. package/dist/tools/executor.js +169 -0
  184. package/dist/tools/index.d.ts +5 -0
  185. package/dist/tools/index.js +5 -0
  186. package/dist/tools/registry.d.ts +21 -0
  187. package/dist/tools/registry.js +75 -0
  188. package/dist/tools/runtime.d.ts +27 -0
  189. package/dist/tools/runtime.js +48 -0
  190. package/dist/tools/schema-validator.d.ts +24 -0
  191. package/dist/tools/schema-validator.js +88 -0
  192. package/dist/tools/types.d.ts +86 -0
  193. package/dist/tools/types.js +1 -0
  194. package/dist/variation-images.d.ts +19 -0
  195. package/dist/variation-images.js +12 -0
  196. package/package.json +78 -0
@@ -0,0 +1,247 @@
1
+ import { createHash } from "node:crypto";
2
+ import { appendFile, mkdir, readFile } from "node:fs/promises";
3
+ import { existsSync } from "node:fs";
4
+ import { resolve } from "node:path";
5
+ import { toErrorDetail as _unifiedToErrorDetail } from "../errors.js";
6
+ // Use the canonical toErrorDetail from errors.ts
7
+ const toErrorDetail = _unifiedToErrorDetail;
8
+ export function createChatTelemetryStore(args) {
9
+ const buffer = [];
10
+ const pendingWrites = [];
11
+ let flushTimer = null;
12
+ function promptExcerpt(message) {
13
+ return message.replace(/\s+/g, " ").trim().slice(0, 180);
14
+ }
15
+ function promptHash(message) {
16
+ return createHash("sha256").update(message).digest("hex").slice(0, 16);
17
+ }
18
+ async function flushNow() {
19
+ if (!args.persistEnabled)
20
+ return;
21
+ if (pendingWrites.length === 0)
22
+ return;
23
+ const pending = pendingWrites.splice(0, pendingWrites.length);
24
+ const lines = pending.map((item) => JSON.stringify(item)).join("\n");
25
+ await mkdir(resolve(args.filePath, ".."), { recursive: true });
26
+ await appendFile(args.filePath, `${lines}\n`, "utf8");
27
+ }
28
+ function scheduleFlush() {
29
+ if (flushTimer)
30
+ clearTimeout(flushTimer);
31
+ flushTimer = setTimeout(() => {
32
+ void flushNow().catch((error) => {
33
+ args.logger.error({ err: toErrorDetail(error), file: args.filePath }, "Failed to flush chat telemetry");
34
+ });
35
+ }, 150);
36
+ }
37
+ async function loadFromDisk() {
38
+ if (!args.persistEnabled)
39
+ return;
40
+ if (!existsSync(args.filePath))
41
+ return;
42
+ try {
43
+ const raw = await readFile(args.filePath, "utf8");
44
+ const lines = raw
45
+ .split("\n")
46
+ .map((line) => line.trim())
47
+ .filter(Boolean);
48
+ const tail = lines.slice(-Math.max(args.limit, 100));
49
+ buffer.length = 0;
50
+ for (const line of tail) {
51
+ try {
52
+ const parsed = JSON.parse(line);
53
+ if (!parsed || typeof parsed !== "object")
54
+ continue;
55
+ if (typeof parsed.id !== "string" || typeof parsed.phase !== "string")
56
+ continue;
57
+ buffer.push(parsed);
58
+ }
59
+ catch {
60
+ // Ignore malformed telemetry lines.
61
+ }
62
+ }
63
+ args.logger.info({ file: args.filePath, loaded: buffer.length }, "Loaded chat telemetry");
64
+ }
65
+ catch (error) {
66
+ args.logger.error({ err: toErrorDetail(error), file: args.filePath }, "Failed to load chat telemetry");
67
+ }
68
+ }
69
+ function push(entry) {
70
+ buffer.push(entry);
71
+ if (buffer.length > args.limit) {
72
+ buffer.splice(0, buffer.length - args.limit);
73
+ }
74
+ if (args.persistEnabled) {
75
+ pendingWrites.push(entry);
76
+ scheduleFlush();
77
+ }
78
+ if (args.onPush) {
79
+ try {
80
+ args.onPush(entry);
81
+ }
82
+ catch (error) {
83
+ args.logger.error({ err: toErrorDetail(error) }, "chat-telemetry onPush hook failed");
84
+ }
85
+ }
86
+ args.logger.info({
87
+ event: "chat_telemetry",
88
+ id: entry.id,
89
+ phase: entry.phase,
90
+ session: entry.session,
91
+ requestedSlug: entry.requestedSlug,
92
+ effectiveSlug: entry.effectiveSlug,
93
+ plannerSource: entry.plannerSource,
94
+ modelKey: entry.modelKey,
95
+ modelUsed: entry.modelUsed,
96
+ promptHash: entry.promptHash,
97
+ promptLength: entry.promptLength,
98
+ outcome: entry.outcome,
99
+ timelineStage: entry.timelineStage,
100
+ reasonCategory: entry.reasonCategory,
101
+ opCount: entry.opCount,
102
+ opTypes: entry.opTypes,
103
+ skippedOpCount: entry.skippedOpCount,
104
+ inputTokens: entry.inputTokens,
105
+ outputTokens: entry.outputTokens,
106
+ totalTokens: entry.totalTokens,
107
+ cacheCreationInputTokens: entry.cacheCreationInputTokens,
108
+ cacheReadInputTokens: entry.cacheReadInputTokens,
109
+ estimatedUsd: entry.estimatedUsd,
110
+ totalDurationMs: entry.totalDurationMs,
111
+ planningDurationMs: entry.planningDurationMs,
112
+ firstPlanningTokenMs: entry.firstPlanningTokenMs,
113
+ applyDurationMs: entry.applyDurationMs,
114
+ imageResolutionDurationMs: entry.imageResolutionDurationMs,
115
+ planningAttempts: entry.planningAttempts,
116
+ contextPackBytes: entry.contextPackBytes,
117
+ contractMode: entry.contractMode,
118
+ contractBytes: entry.contractBytes,
119
+ contractBlockCount: entry.contractBlockCount,
120
+ strictJsonEnabled: entry.strictJsonEnabled,
121
+ schemaRetryUsed: entry.schemaRetryUsed,
122
+ plannerRefusal: entry.plannerRefusal,
123
+ plannerIncomplete: entry.plannerIncomplete,
124
+ compactContextEnabled: entry.compactContextEnabled,
125
+ minimalContextEnabled: entry.minimalContextEnabled,
126
+ plannerTier: entry.plannerTier,
127
+ toolName: entry.toolName,
128
+ toolOk: entry.toolOk,
129
+ toolLatencyMs: entry.toolLatencyMs,
130
+ toolAttempts: entry.toolAttempts,
131
+ toolErrorCode: entry.toolErrorCode,
132
+ correlationId: entry.correlationId
133
+ }, "Chat telemetry event");
134
+ }
135
+ function list(args) {
136
+ const limitRaw = Number(args.limit ?? 100);
137
+ const limit = Number.isFinite(limitRaw) && limitRaw > 0 ? Math.min(Math.trunc(limitRaw), 1000) : 100;
138
+ let rows = buffer;
139
+ if (args.outcome)
140
+ rows = rows.filter((row) => row.outcome === args.outcome);
141
+ if (args.phase)
142
+ rows = rows.filter((row) => row.phase === args.phase);
143
+ if (args.session)
144
+ rows = rows.filter((row) => row.session === args.session);
145
+ const recent = rows.slice(Math.max(0, rows.length - limit));
146
+ const byOutcome = {};
147
+ const byReasonCategory = {};
148
+ for (const row of recent) {
149
+ if (row.outcome)
150
+ byOutcome[row.outcome] = (byOutcome[row.outcome] ?? 0) + 1;
151
+ if (row.reasonCategory)
152
+ byReasonCategory[row.reasonCategory] = (byReasonCategory[row.reasonCategory] ?? 0) + 1;
153
+ }
154
+ return {
155
+ totalBuffered: buffer.length,
156
+ returned: recent.length,
157
+ byOutcome,
158
+ byReasonCategory,
159
+ rows: recent
160
+ };
161
+ }
162
+ function review(args) {
163
+ const limitRaw = Number(args.limit ?? 300);
164
+ const limit = Number.isFinite(limitRaw) && limitRaw > 0 ? Math.min(Math.trunc(limitRaw), 2000) : 300;
165
+ let rows = buffer;
166
+ if (args.session)
167
+ rows = rows.filter((row) => row.session === args.session);
168
+ const recent = rows.slice(Math.max(0, rows.length - limit));
169
+ const failureOutcomes = new Set([
170
+ "guardrail_failure",
171
+ "apply_failed",
172
+ "repair_failed",
173
+ "planner_exception",
174
+ "planning_exhausted",
175
+ "planning_refusal",
176
+ "planning_incomplete",
177
+ "planning_missing"
178
+ ]);
179
+ const failures = recent.filter((row) => row.phase === "result" && row.outcome && failureOutcomes.has(row.outcome));
180
+ const success = recent.filter((row) => row.phase === "result" && row.outcome === "applied");
181
+ const failureByReasonCategory = {};
182
+ const failureByOutcome = {};
183
+ let plannerRefusalCount = 0;
184
+ let plannerIncompleteCount = 0;
185
+ const byPromptHash = new Map();
186
+ for (const row of failures) {
187
+ if (row.plannerRefusal)
188
+ plannerRefusalCount += 1;
189
+ if (row.plannerIncomplete)
190
+ plannerIncompleteCount += 1;
191
+ if (row.reasonCategory)
192
+ failureByReasonCategory[row.reasonCategory] = (failureByReasonCategory[row.reasonCategory] ?? 0) + 1;
193
+ if (row.outcome)
194
+ failureByOutcome[row.outcome] = (failureByOutcome[row.outcome] ?? 0) + 1;
195
+ const current = byPromptHash.get(row.promptHash) ??
196
+ { promptExcerpt: row.promptExcerpt, count: 0, outcomes: {}, reasonCategories: {}, lastAt: row.at };
197
+ current.count += 1;
198
+ if (row.outcome)
199
+ current.outcomes[row.outcome] = (current.outcomes[row.outcome] ?? 0) + 1;
200
+ if (row.reasonCategory)
201
+ current.reasonCategories[row.reasonCategory] = (current.reasonCategories[row.reasonCategory] ?? 0) + 1;
202
+ if (row.at > current.lastAt)
203
+ current.lastAt = row.at;
204
+ byPromptHash.set(row.promptHash, current);
205
+ }
206
+ const topFailedPrompts = Array.from(byPromptHash.entries())
207
+ .map(([promptHash, value]) => ({ promptHash, ...value }))
208
+ .sort((a, b) => b.count - a.count)
209
+ .slice(0, 12);
210
+ const recommendations = [];
211
+ if ((failureByReasonCategory.schema_violation ?? 0) > 0) {
212
+ recommendations.push("Schema violations are frequent: add stricter pre-apply normalization for missing required fields and alias keys.");
213
+ }
214
+ if ((failureByReasonCategory.not_found ?? 0) > 0) {
215
+ recommendations.push("Not-found failures detected: improve slug/block resolution using active selection and current page context.");
216
+ }
217
+ if ((failureByReasonCategory.ambiguity ?? 0) > 0) {
218
+ recommendations.push("Ambiguity is high: improve follow-up question templates with explicit selectable options.");
219
+ }
220
+ if ((failureByOutcome.planning_exhausted ?? 0) > 0) {
221
+ recommendations.push("Planner retries are exhausting: add deterministic fallback plans for the top failed prompt families.");
222
+ }
223
+ if (recommendations.length === 0) {
224
+ recommendations.push("No dominant failure mode detected in this sample. Review top failed prompts and add targeted tests for each.");
225
+ }
226
+ return {
227
+ analyzed: recent.length,
228
+ appliedCount: success.length,
229
+ failedCount: failures.length,
230
+ failureRate: recent.length > 0 ? Number((failures.length / recent.length).toFixed(4)) : 0,
231
+ plannerRefusalCount,
232
+ plannerIncompleteCount,
233
+ failureByOutcome,
234
+ failureByReasonCategory,
235
+ topFailedPrompts,
236
+ recommendations
237
+ };
238
+ }
239
+ return {
240
+ promptExcerpt,
241
+ promptHash,
242
+ push,
243
+ loadFromDisk,
244
+ list,
245
+ review
246
+ };
247
+ }
@@ -0,0 +1,50 @@
1
+ import type { PageDoc } from "@avocadostudio-ai/shared";
2
+ export type EvalCandidate = {
3
+ id: string;
4
+ at: string;
5
+ session: string;
6
+ slug: string;
7
+ activeBlockId?: string;
8
+ activeEditablePath?: string;
9
+ prompt: string;
10
+ fixture: PageDoc[];
11
+ outcome?: string;
12
+ reasonCategory?: string;
13
+ plannerTier?: string;
14
+ opTypes?: string[];
15
+ opCount?: number;
16
+ provider?: string;
17
+ modelKey?: string;
18
+ modelUsed?: string;
19
+ };
20
+ export type EvalCandidateOpen = Omit<EvalCandidate, "at" | "outcome" | "reasonCategory" | "plannerTier" | "opTypes" | "opCount">;
21
+ export type EvalCandidateFinalize = Partial<Pick<EvalCandidate, "outcome" | "reasonCategory" | "plannerTier" | "opTypes" | "opCount">>;
22
+ type Logger = {
23
+ info: (payload: Record<string, unknown>, message?: string) => void;
24
+ error: (payload: Record<string, unknown>, message?: string) => void;
25
+ };
26
+ type CreateEvalCandidateStoreArgs = {
27
+ filePath: string;
28
+ limit: number;
29
+ persistEnabled: boolean;
30
+ ttlDays: number;
31
+ logger: Logger;
32
+ };
33
+ export declare function createEvalCandidateStore(args: CreateEvalCandidateStoreArgs): {
34
+ start: (entry: EvalCandidateOpen) => void;
35
+ finalize: (id: string, patch: EvalCandidateFinalize) => EvalCandidate | undefined;
36
+ get: (id: string) => EvalCandidate | undefined;
37
+ list: (filters: {
38
+ limit?: number;
39
+ session?: string;
40
+ outcome?: string;
41
+ }) => {
42
+ total: number;
43
+ rows: EvalCandidate[];
44
+ };
45
+ cancel: (id: string) => void;
46
+ loadFromDisk: () => Promise<void>;
47
+ flushNow: () => Promise<void>;
48
+ };
49
+ export type EvalCandidateStore = ReturnType<typeof createEvalCandidateStore>;
50
+ export {};
@@ -0,0 +1,120 @@
1
+ import { appendFile, mkdir, readFile } from "node:fs/promises";
2
+ import { existsSync } from "node:fs";
3
+ import { resolve } from "node:path";
4
+ import { toErrorDetail as _unifiedToErrorDetail } from "../errors.js";
5
+ const toErrorDetail = _unifiedToErrorDetail;
6
+ export function createEvalCandidateStore(args) {
7
+ const open = new Map();
8
+ const finalized = [];
9
+ const pendingWrites = [];
10
+ let flushTimer = null;
11
+ async function flushNow() {
12
+ if (!args.persistEnabled)
13
+ return;
14
+ if (pendingWrites.length === 0)
15
+ return;
16
+ const pending = pendingWrites.splice(0, pendingWrites.length);
17
+ const lines = pending.map((item) => JSON.stringify(item)).join("\n");
18
+ await mkdir(resolve(args.filePath, ".."), { recursive: true });
19
+ await appendFile(args.filePath, `${lines}\n`, "utf8");
20
+ }
21
+ function scheduleFlush() {
22
+ if (flushTimer)
23
+ clearTimeout(flushTimer);
24
+ flushTimer = setTimeout(() => {
25
+ void flushNow().catch((error) => {
26
+ args.logger.error({ err: toErrorDetail(error), file: args.filePath }, "Failed to flush eval candidates");
27
+ });
28
+ }, 150);
29
+ }
30
+ function isWithinTtl(isoAt, now, ttlMs) {
31
+ const t = Date.parse(isoAt);
32
+ if (!Number.isFinite(t))
33
+ return false;
34
+ return now - t <= ttlMs;
35
+ }
36
+ async function loadFromDisk() {
37
+ if (!args.persistEnabled)
38
+ return;
39
+ if (!existsSync(args.filePath))
40
+ return;
41
+ try {
42
+ const raw = await readFile(args.filePath, "utf8");
43
+ const lines = raw.split("\n").map((line) => line.trim()).filter(Boolean);
44
+ const now = Date.now();
45
+ const ttlMs = args.ttlDays * 24 * 60 * 60 * 1000;
46
+ finalized.length = 0;
47
+ for (const line of lines) {
48
+ try {
49
+ const parsed = JSON.parse(line);
50
+ if (!parsed || typeof parsed !== "object")
51
+ continue;
52
+ if (typeof parsed.id !== "string" || typeof parsed.prompt !== "string")
53
+ continue;
54
+ if (!isWithinTtl(parsed.at, now, ttlMs))
55
+ continue;
56
+ finalized.push(parsed);
57
+ }
58
+ catch {
59
+ // ignore malformed lines
60
+ }
61
+ }
62
+ if (finalized.length > args.limit) {
63
+ finalized.splice(0, finalized.length - args.limit);
64
+ }
65
+ args.logger.info({ file: args.filePath, loaded: finalized.length }, "Loaded eval candidates");
66
+ }
67
+ catch (error) {
68
+ args.logger.error({ err: toErrorDetail(error), file: args.filePath }, "Failed to load eval candidates");
69
+ }
70
+ }
71
+ function start(entry) {
72
+ open.set(entry.id, entry);
73
+ }
74
+ function finalize(id, patch) {
75
+ const opened = open.get(id);
76
+ if (!opened)
77
+ return undefined;
78
+ open.delete(id);
79
+ const record = { ...opened, ...patch, at: new Date().toISOString() };
80
+ finalized.push(record);
81
+ if (finalized.length > args.limit) {
82
+ finalized.splice(0, finalized.length - args.limit);
83
+ }
84
+ if (args.persistEnabled) {
85
+ pendingWrites.push(record);
86
+ scheduleFlush();
87
+ }
88
+ args.logger.info({
89
+ event: "eval_candidate",
90
+ id: record.id,
91
+ session: record.session,
92
+ slug: record.slug,
93
+ outcome: record.outcome,
94
+ opTypes: record.opTypes,
95
+ opCount: record.opCount,
96
+ plannerTier: record.plannerTier
97
+ }, "Eval candidate finalized");
98
+ return record;
99
+ }
100
+ function get(id) {
101
+ for (let i = finalized.length - 1; i >= 0; i--) {
102
+ if (finalized[i].id === id)
103
+ return finalized[i];
104
+ }
105
+ return undefined;
106
+ }
107
+ function list(filters) {
108
+ const max = Math.min(Math.max(Number(filters.limit) || 100, 1), 1000);
109
+ let rows = finalized;
110
+ if (filters.session)
111
+ rows = rows.filter((r) => r.session === filters.session);
112
+ if (filters.outcome)
113
+ rows = rows.filter((r) => r.outcome === filters.outcome);
114
+ return { total: rows.length, rows: rows.slice(-max) };
115
+ }
116
+ function cancel(id) {
117
+ open.delete(id);
118
+ }
119
+ return { start, finalize, get, list, cancel, loadFromDisk, flushNow };
120
+ }
@@ -0,0 +1,34 @@
1
+ export type FeedbackRating = "up" | "down";
2
+ export type FeedbackEntry = {
3
+ id: string;
4
+ at: string;
5
+ traceId: string;
6
+ session: string;
7
+ rating: FeedbackRating;
8
+ note?: string;
9
+ };
10
+ type Logger = {
11
+ info: (payload: Record<string, unknown>, message?: string) => void;
12
+ error: (payload: Record<string, unknown>, message?: string) => void;
13
+ };
14
+ type CreateFeedbackStoreArgs = {
15
+ filePath: string;
16
+ limit: number;
17
+ logger: Logger;
18
+ };
19
+ export declare function createFeedbackStore(args: CreateFeedbackStoreArgs): {
20
+ push: (entry: FeedbackEntry) => void;
21
+ list: (filters: {
22
+ session?: string;
23
+ rating?: string;
24
+ traceId?: string;
25
+ limit?: number;
26
+ }) => {
27
+ total: number;
28
+ rows: FeedbackEntry[];
29
+ };
30
+ loadFromDisk: () => Promise<void>;
31
+ flushNow: () => Promise<void>;
32
+ };
33
+ export type FeedbackStore = ReturnType<typeof createFeedbackStore>;
34
+ export {};
@@ -0,0 +1,76 @@
1
+ import { appendFile, mkdir, readFile } from "node:fs/promises";
2
+ import { existsSync } from "node:fs";
3
+ import { resolve } from "node:path";
4
+ export function createFeedbackStore(args) {
5
+ const buffer = [];
6
+ const pendingWrites = [];
7
+ let flushTimer = null;
8
+ async function flushNow() {
9
+ if (pendingWrites.length === 0)
10
+ return;
11
+ const pending = pendingWrites.splice(0, pendingWrites.length);
12
+ const lines = pending.map((item) => JSON.stringify(item)).join("\n");
13
+ await mkdir(resolve(args.filePath, ".."), { recursive: true });
14
+ await appendFile(args.filePath, `${lines}\n`, "utf8");
15
+ }
16
+ function scheduleFlush() {
17
+ if (flushTimer)
18
+ clearTimeout(flushTimer);
19
+ flushTimer = setTimeout(() => {
20
+ void flushNow().catch((error) => {
21
+ args.logger.error({ err: String(error), file: args.filePath }, "Failed to flush feedback");
22
+ });
23
+ }, 150);
24
+ }
25
+ async function loadFromDisk() {
26
+ if (!existsSync(args.filePath))
27
+ return;
28
+ try {
29
+ const raw = await readFile(args.filePath, "utf8");
30
+ const lines = raw.split("\n").map((line) => line.trim()).filter(Boolean);
31
+ const tail = lines.slice(-args.limit);
32
+ buffer.length = 0;
33
+ for (const line of tail) {
34
+ try {
35
+ const parsed = JSON.parse(line);
36
+ if (!parsed || typeof parsed !== "object")
37
+ continue;
38
+ if (typeof parsed.id !== "string" || typeof parsed.rating !== "string")
39
+ continue;
40
+ buffer.push(parsed);
41
+ }
42
+ catch {
43
+ // Ignore malformed lines.
44
+ }
45
+ }
46
+ args.logger.info({ file: args.filePath, loaded: buffer.length }, "Loaded feedback entries");
47
+ }
48
+ catch (error) {
49
+ args.logger.error({ err: String(error), file: args.filePath }, "Failed to load feedback");
50
+ }
51
+ }
52
+ function push(entry) {
53
+ buffer.push(entry);
54
+ if (buffer.length > args.limit) {
55
+ buffer.splice(0, buffer.length - args.limit);
56
+ }
57
+ pendingWrites.push(entry);
58
+ scheduleFlush();
59
+ args.logger.info({ event: "feedback", id: entry.id, traceId: entry.traceId, session: entry.session, rating: entry.rating }, "Feedback recorded");
60
+ }
61
+ function list(filters) {
62
+ const max = Math.min(Math.max(Number(filters.limit) || 200, 1), 1000);
63
+ let rows = buffer;
64
+ if (filters.session)
65
+ rows = rows.filter((r) => r.session === filters.session);
66
+ if (filters.rating)
67
+ rows = rows.filter((r) => r.rating === filters.rating);
68
+ if (filters.traceId)
69
+ rows = rows.filter((r) => r.traceId === filters.traceId);
70
+ return {
71
+ total: rows.length,
72
+ rows: rows.slice(-max)
73
+ };
74
+ }
75
+ return { push, list, loadFromDisk, flushNow };
76
+ }
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Jira ticket telemetry — persists per-run traces to JSONL so each ticket
3
+ * can be reviewed end-to-end after a test (instruction → tool calls → summary).
4
+ * Follows the append-flush pattern of chat-telemetry.ts / migration-telemetry.ts.
5
+ *
6
+ * Location: ~/.data/telemetry/jira-telemetry.jsonl
7
+ */
8
+ export type JiraToolCallTrace = {
9
+ name: string;
10
+ /** Redacted/truncated input — see redactToolInput */
11
+ input: unknown;
12
+ durationMs: number;
13
+ /** First ~400 chars of the tool's string result, for quick scan */
14
+ resultExcerpt: string;
15
+ isError: boolean;
16
+ };
17
+ export type JiraTelemetryEntry = {
18
+ timestamp: string;
19
+ issueKey: string;
20
+ mode: "review" | "execute" | "publish";
21
+ siteId: string;
22
+ session: string;
23
+ status: "success" | "error";
24
+ durationMs: number;
25
+ model?: string;
26
+ provider?: string;
27
+ /** Full instruction string sent to the agent (after comment history is appended) */
28
+ instruction?: string;
29
+ /** Review-mode only: parsed JSON verdict */
30
+ reviewDecision?: {
31
+ decision: "proceed" | "questions";
32
+ plan: string[];
33
+ questions?: string[];
34
+ };
35
+ /** Execute-mode only: tool-by-tool trace */
36
+ toolCalls?: JiraToolCallTrace[];
37
+ /** Number of `tool_done` events the loop saw (matches pino log totals) */
38
+ toolCallCount?: number;
39
+ /** Final summary text the agent returned (execute mode) */
40
+ summary?: string;
41
+ /** Changes array (execute mode) — one string per applied op group */
42
+ changes?: string[];
43
+ /** Slugs touched (execute mode) */
44
+ touchedSlugs?: string[];
45
+ /** True when auto-publish or publish mode succeeded */
46
+ published?: boolean;
47
+ /** Status transitions attempted during the run */
48
+ transitions?: string[];
49
+ /** Error message (status=error) */
50
+ error?: string;
51
+ };
52
+ export declare function pushJiraTelemetry(entry: JiraTelemetryEntry): void;
53
+ /**
54
+ * Make an object safe for JSONL — shallow clone and truncate large string
55
+ * values. Preserves structure so analysis scripts can still inspect shape.
56
+ */
57
+ export declare function redactToolInput(input: unknown, maxStringChars?: number): unknown;
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Jira ticket telemetry — persists per-run traces to JSONL so each ticket
3
+ * can be reviewed end-to-end after a test (instruction → tool calls → summary).
4
+ * Follows the append-flush pattern of chat-telemetry.ts / migration-telemetry.ts.
5
+ *
6
+ * Location: ~/.data/telemetry/jira-telemetry.jsonl
7
+ */
8
+ import { appendFile, mkdir } from "node:fs/promises";
9
+ import { resolve, dirname } from "node:path";
10
+ import { homedir } from "node:os";
11
+ const TELEMETRY_FILE = resolve(homedir(), ".data/telemetry/jira-telemetry.jsonl");
12
+ const pendingWrites = [];
13
+ let flushTimer = null;
14
+ async function flushNow() {
15
+ if (pendingWrites.length === 0)
16
+ return;
17
+ const pending = pendingWrites.splice(0, pendingWrites.length);
18
+ const lines = pending.map((item) => JSON.stringify(item)).join("\n");
19
+ await mkdir(dirname(TELEMETRY_FILE), { recursive: true });
20
+ await appendFile(TELEMETRY_FILE, `${lines}\n`, "utf8");
21
+ }
22
+ function scheduleFlush() {
23
+ if (flushTimer)
24
+ clearTimeout(flushTimer);
25
+ flushTimer = setTimeout(() => {
26
+ void flushNow().catch((err) => {
27
+ console.error("[jira-telemetry] flush failed:", err instanceof Error ? err.message : String(err));
28
+ });
29
+ }, 150);
30
+ flushTimer.unref();
31
+ }
32
+ process.on("beforeExit", () => { void flushNow().catch(() => { }); });
33
+ export function pushJiraTelemetry(entry) {
34
+ pendingWrites.push(entry);
35
+ scheduleFlush();
36
+ // Console summary so tail -f on stdout reads well during live testing.
37
+ const toolPart = entry.toolCalls && entry.toolCalls.length > 0
38
+ ? ` | tools: ${entry.toolCalls.map((t) => t.name).join(",")}`
39
+ : "";
40
+ const decisionPart = entry.reviewDecision ? ` | ${entry.reviewDecision.decision}` : "";
41
+ console.log(`[jira-telemetry] ${entry.issueKey} ${entry.mode} ${entry.status} ` +
42
+ `| ${(entry.durationMs / 1000).toFixed(1)}s${decisionPart}${toolPart}` +
43
+ (entry.error ? ` | err: ${entry.error.slice(0, 120)}` : ""));
44
+ }
45
+ /**
46
+ * Make an object safe for JSONL — shallow clone and truncate large string
47
+ * values. Preserves structure so analysis scripts can still inspect shape.
48
+ */
49
+ export function redactToolInput(input, maxStringChars = 1000) {
50
+ if (input == null || typeof input !== "object")
51
+ return input;
52
+ if (Array.isArray(input)) {
53
+ return input.slice(0, 50).map((v) => redactToolInput(v, maxStringChars));
54
+ }
55
+ const out = {};
56
+ for (const [k, v] of Object.entries(input)) {
57
+ if (typeof v === "string" && v.length > maxStringChars) {
58
+ out[k] = `${v.slice(0, maxStringChars)}… [truncated ${v.length - maxStringChars} chars]`;
59
+ }
60
+ else if (typeof v === "object" && v !== null) {
61
+ out[k] = redactToolInput(v, maxStringChars);
62
+ }
63
+ else {
64
+ out[k] = v;
65
+ }
66
+ }
67
+ return out;
68
+ }