@proofhound/core 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/common/contracts/index.d.ts +1 -0
- package/dist/server/common/contracts/index.d.ts.map +1 -1
- package/dist/server/common/contracts/index.js +1 -0
- package/dist/server/common/contracts/index.js.map +1 -1
- package/dist/server/common/contracts/local-contracts.module.d.ts.map +1 -1
- package/dist/server/common/contracts/local-contracts.module.js +3 -0
- package/dist/server/common/contracts/local-contracts.module.js.map +1 -1
- package/dist/server/common/contracts/usage-metering.hook.d.ts +33 -0
- package/dist/server/common/contracts/usage-metering.hook.d.ts.map +1 -0
- package/dist/server/common/contracts/usage-metering.hook.js +43 -0
- package/dist/server/common/contracts/usage-metering.hook.js.map +1 -0
- package/dist/server/infrastructure/llm/run-result-writer.d.ts +4 -1
- package/dist/server/infrastructure/llm/run-result-writer.d.ts.map +1 -1
- package/dist/server/infrastructure/llm/run-result-writer.js +63 -11
- package/dist/server/infrastructure/llm/run-result-writer.js.map +1 -1
- package/dist/server/modules/dataset/dataset-import.service.d.ts.map +1 -1
- package/dist/server/modules/dataset/dataset-import.service.js +7 -0
- package/dist/server/modules/dataset/dataset-import.service.js.map +1 -1
- package/dist/server/modules/dataset/dataset.service.d.ts +13 -1
- package/dist/server/modules/dataset/dataset.service.d.ts.map +1 -1
- package/dist/server/modules/dataset/dataset.service.js +85 -2
- package/dist/server/modules/dataset/dataset.service.js.map +1 -1
- package/dist/server/modules/model/model.service.d.ts +5 -1
- package/dist/server/modules/model/model.service.d.ts.map +1 -1
- package/dist/server/modules/model/model.service.js +65 -3
- package/dist/server/modules/model/model.service.js.map +1 -1
- package/dist/server/modules/release-line/release-line.service.d.ts +4 -1
- package/dist/server/modules/release-line/release-line.service.d.ts.map +1 -1
- package/dist/server/modules/release-line/release-line.service.js +83 -3
- package/dist/server/modules/release-line/release-line.service.js.map +1 -1
- package/dist/server/modules/release-line/release-runner.service.d.ts +4 -1
- package/dist/server/modules/release-line/release-runner.service.d.ts.map +1 -1
- package/dist/server/modules/release-line/release-runner.service.js +35 -2
- package/dist/server/modules/release-line/release-runner.service.js.map +1 -1
- package/dist/worker/consumers/llm.consumer.d.ts +4 -1
- package/dist/worker/consumers/llm.consumer.d.ts.map +1 -1
- package/dist/worker/consumers/llm.consumer.js +38 -3
- package/dist/worker/consumers/llm.consumer.js.map +1 -1
- package/dist/worker/consumers/probe.consumer.d.ts +4 -1
- package/dist/worker/consumers/probe.consumer.d.ts.map +1 -1
- package/dist/worker/consumers/probe.consumer.js +35 -3
- package/dist/worker/consumers/probe.consumer.js.map +1 -1
- package/dist/worker/runners/llm-runner.d.ts +3 -1
- package/dist/worker/runners/llm-runner.d.ts.map +1 -1
- package/dist/worker/runners/llm-runner.js +102 -55
- package/dist/worker/runners/llm-runner.js.map +1 -1
- package/dist/worker/runners/probe-runner.d.ts +9 -2
- package/dist/worker/runners/probe-runner.d.ts.map +1 -1
- package/dist/worker/runners/probe-runner.js +46 -2
- package/dist/worker/runners/probe-runner.js.map +1 -1
- package/dist/worker/runners/run-result-writer.d.ts +4 -1
- package/dist/worker/runners/run-result-writer.d.ts.map +1 -1
- package/dist/worker/runners/run-result-writer.js +60 -10
- package/dist/worker/runners/run-result-writer.js.map +1 -1
- package/package.json +13 -13
|
@@ -7,13 +7,32 @@ const node_crypto_1 = require("node:crypto");
|
|
|
7
7
|
const drizzle_orm_1 = require("drizzle-orm");
|
|
8
8
|
const db_1 = require("@proofhound/db");
|
|
9
9
|
const judgment_1 = require("@proofhound/judgment");
|
|
10
|
+
const limiter_1 = require("@proofhound/limiter");
|
|
10
11
|
const llm_client_1 = require("@proofhound/llm-client");
|
|
12
|
+
const usage_metering_hook_1 = require("../../server/common/contracts/usage-metering.hook");
|
|
11
13
|
const runtime_limits_1 = require("../../shared/llm/runtime-limits");
|
|
12
14
|
const run_result_writer_1 = require("./run-result-writer");
|
|
13
15
|
function createLlmRunner(deps) {
|
|
14
|
-
const runResultWriter = new run_result_writer_1.DrizzleRunResultWriter(deps.db, deps.quotaPolicy);
|
|
16
|
+
const runResultWriter = new run_result_writer_1.DrizzleRunResultWriter(deps.db, deps.quotaPolicy, deps.usageMetering);
|
|
15
17
|
return async function runLlmJob(input, jobContext) {
|
|
16
18
|
const runResultId = input.runResultId ?? (0, node_crypto_1.randomUUID)();
|
|
19
|
+
const basePayload = {
|
|
20
|
+
queue: jobContext.bullmqQueue,
|
|
21
|
+
jobId: jobContext.bullmqJobId,
|
|
22
|
+
attempt: jobContext.attempt,
|
|
23
|
+
runResultId,
|
|
24
|
+
modelId: input.modelId,
|
|
25
|
+
source: input.source,
|
|
26
|
+
};
|
|
27
|
+
const recordJobEvent = (eventType, payload) => (0, usage_metering_hook_1.safeRecordUsageEvent)(deps.usageMetering, {
|
|
28
|
+
idempotencyKey: `job:${jobContext.bullmqQueue}:${jobContext.bullmqJobId}:${jobContext.attempt}:${eventType}`,
|
|
29
|
+
dimension: 'job',
|
|
30
|
+
eventType,
|
|
31
|
+
projectId: input.projectId,
|
|
32
|
+
occurredAt: new Date(),
|
|
33
|
+
source: 'worker',
|
|
34
|
+
payload: { ...basePayload, ...payload },
|
|
35
|
+
}, deps.logger);
|
|
17
36
|
const model = await loadModelInvocationConfig(deps, input.modelId);
|
|
18
37
|
// Fold any deployment-level runtime caps (a SaaS org plan's ceiling, SPEC 08 §3.10) into the per-call limits at the
|
|
19
38
|
// single worker enforcement point, so every job source (experiment / optimization child / release / webhook) is
|
|
@@ -41,60 +60,88 @@ function createLlmRunner(deps) {
|
|
|
41
60
|
// Build the rate-limit key at the runtime layer (§3.7); llm-client/limiter stay project-unaware (§8).
|
|
42
61
|
const project = { projectId: input.projectId, orgId: input.orgId, source: 'local' };
|
|
43
62
|
const limiterKey = deps.limiterKeyStrategy.buildModelKey(project, input.modelId);
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
63
|
+
let result;
|
|
64
|
+
try {
|
|
65
|
+
result = await deps.quotaPolicy.withExecutionSlot({ project, source: input.source, modelId: input.modelId, requestId: input.requestId }, () => (0, llm_client_1.invokeLLM)({
|
|
66
|
+
model: effectiveModel,
|
|
67
|
+
limiterKey,
|
|
68
|
+
messages: input.renderedPrompt.messages,
|
|
69
|
+
prompt: input.renderedPrompt.prompt,
|
|
70
|
+
params: {
|
|
71
|
+
temperature: input.inference?.temperature,
|
|
72
|
+
maxTokens: input.inference?.maxTokens,
|
|
73
|
+
topP: input.inference?.topP,
|
|
74
|
+
tools: input.renderedPrompt.tools,
|
|
75
|
+
responseFormat: input.renderedPrompt.responseFormat,
|
|
76
|
+
imageRefs: input.renderedPrompt.imageRefs,
|
|
77
|
+
apiVersion: input.inference?.apiVersion,
|
|
78
|
+
},
|
|
79
|
+
maxRetries: input.retry?.maxRetries,
|
|
80
|
+
context: {
|
|
81
|
+
requestId: input.requestId,
|
|
82
|
+
dbosWorkflowId: jobContext.dbosWorkflowId,
|
|
83
|
+
bullmqJobId: jobContext.bullmqJobId,
|
|
84
|
+
bullmqQueue: jobContext.bullmqQueue,
|
|
85
|
+
stepName: jobContext.stepName,
|
|
86
|
+
runResultId,
|
|
87
|
+
promptId: input.promptId,
|
|
88
|
+
promptVersionId: input.promptVersionId,
|
|
89
|
+
source: input.source,
|
|
90
|
+
attempt: jobContext.attempt,
|
|
91
|
+
},
|
|
92
|
+
runResult: {
|
|
93
|
+
id: runResultId,
|
|
94
|
+
projectId: input.projectId,
|
|
95
|
+
source: input.source,
|
|
96
|
+
sourceId: input.sourceId,
|
|
97
|
+
releaseVariantId: input.releaseVariantId ?? null,
|
|
98
|
+
promptVersionId: input.promptVersionId,
|
|
99
|
+
modelId: input.modelId,
|
|
100
|
+
sampleId: input.sampleId ?? null,
|
|
101
|
+
externalId: input.externalId ?? null,
|
|
102
|
+
renderedPrompt: normalizeRenderedPrompt(input.renderedPrompt),
|
|
103
|
+
inputVariables: input.inputVariables,
|
|
104
|
+
expectedOutput: expectedOutputAsString(expectedOutput),
|
|
105
|
+
dbosWorkflowId: jobContext.dbosWorkflowId,
|
|
106
|
+
bullmqJobId: jobContext.bullmqJobId,
|
|
107
|
+
attempt: jobContext.attempt,
|
|
108
|
+
webhookTokenId: input.webhookTokenId ?? null,
|
|
109
|
+
},
|
|
110
|
+
// The judgment strategy expects a parsed[expected_field]-style structure; when parseResponse is not provided, parsed=undefined,
|
|
111
|
+
// and the whole metrics is unreliable. Parse strict JSON first; on failure, fall back to parsing a Markdown JSON fence.
|
|
112
|
+
parseResponse: llm_client_1.parseJsonResponseWithMarkdownFallback,
|
|
113
|
+
evaluateJudgment: evaluateJudgmentHook,
|
|
114
|
+
}, {
|
|
115
|
+
limiter: deps.limiter,
|
|
116
|
+
logger: deps.logger,
|
|
117
|
+
runResultWriter,
|
|
118
|
+
onLimiterAcquired: (context) => {
|
|
119
|
+
const acquireResult = context.acquireResult && typeof context.acquireResult === 'object' ? context.acquireResult : null;
|
|
120
|
+
return recordJobEvent('job.started', {
|
|
121
|
+
status: 'started',
|
|
122
|
+
limiterKey: context.key,
|
|
123
|
+
estimatedTokens: context.estimatedTokens,
|
|
124
|
+
effectiveConcurrency: acquireResult?.effectiveConcurrency ?? null,
|
|
125
|
+
});
|
|
126
|
+
},
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
if (error instanceof limiter_1.RateLimitExceededError)
|
|
131
|
+
throw error;
|
|
132
|
+
await recordJobEvent('job.attempt_failed', {
|
|
133
|
+
status: 'failed',
|
|
134
|
+
errorKind: error instanceof Error ? error.name : 'Error',
|
|
135
|
+
});
|
|
136
|
+
throw error;
|
|
137
|
+
}
|
|
138
|
+
await recordJobEvent('job.completed', {
|
|
139
|
+
status: 'completed',
|
|
140
|
+
latencyMs: result.durationMs,
|
|
141
|
+
inputTokens: result.usage.inputTokens,
|
|
142
|
+
outputTokens: result.usage.outputTokens,
|
|
143
|
+
costEstimate: result.costEstimate,
|
|
144
|
+
});
|
|
98
145
|
return {
|
|
99
146
|
runResultId,
|
|
100
147
|
content: result.content,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-runner.js","sourceRoot":"","sources":["../../../src/worker/runners/llm-runner.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"llm-runner.js","sourceRoot":"","sources":["../../../src/worker/runners/llm-runner.ts"],"names":[],"mappings":";;AAwDA,0CA+JC;AAED,8DAyBC;AAED,sDAKC;AAzPD,6CAAyC;AACzC,6CAAiC;AAEjC,uCAAwC;AACxC,mDAAwD;AACxD,iDAA+E;AAC/E,uDASgC;AAKhC,2FAAiH;AACjH,oEAAqE;AAErE,2DAA6D;AAiC7D,SAAgB,eAAe,CAAC,IAA2B;IACzD,MAAM,eAAe,GAAG,IAAI,0CAAsB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;IAElG,OAAO,KAAK,UAAU,SAAS,CAAC,KAAoB,EAAE,UAA+B;QACnF,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,IAAA,wBAAU,GAAE,CAAC;QACtD,MAAM,WAAW,GAAG;YAClB,KAAK,EAAE,UAAU,CAAC,WAAW;YAC7B,KAAK,EAAE,UAAU,CAAC,WAAW;YAC7B,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,WAAW;YACX,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC;QACF,MAAM,cAAc,GAAG,CAAC,SAAiB,EAAE,OAAgC,EAAE,EAAE,CAC7E,IAAA,0CAAoB,EAClB,IAAI,CAAC,aAAa,EAClB;YACE,cAAc,EAAE,OAAO,UAAU,CAAC,WAAW,IAAI,UAAU,CAAC,WAAW,IAAI,UAAU,CAAC,OAAO,IAAI,SAAS,EAAE;YAC5G,SAAS,EAAE,KAAK;YAChB,SAAS;YACT,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,EAAE,GAAG,WAAW,EAAE,GAAG,OAAO,EAAE;SACxC,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;QACJ,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACnE,oHAAoH;QACpH,gHAAgH;QAChH,sGAAsG;QACtG,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,cAAc,CAAC;YACnE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE;YAC5E,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC,CAAC;QACH,4GAA4G;QAC5G,kGAAkG;QAClG,MAAM,cAAc,GAAG,qBAAqB,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QAElE,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,EAAE,cAAc,IAAI,IAAI,CAAC;QAC9D,MAAM,oBAAoB,GAAG,KAAK,CAAC,QAAQ;YACzC,CAAC,CAAC,CAAC,EAAE,MAAM,EAA4C,EAAsB,EAAE;gBAC3E,MAAM,OAAO,GAAG,IAAA,2BAAgB,EAAC,gBAAgB,EAAE,MAAM,EAAE;oBACzD,YAAY,EAAE,KAAK,CAAC,QAAS,CAAC,YAAY;oBAC1C,aAAa,EAAE,KAAK,CAAC,QAAS,CAAC,aAAa;oBAC5C,cAAc,EAAE,KAAK,CAAC,QAAS,CAAC,cAAc;iBAC/C,CAAC,CAAC;gBACH,OAAO,OAAO,CAAC;YACjB,CAAC;YACH,CAAC,CAAC,SAAS,CAAC;QAEd,sGAAsG;QACtG,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAgB,EAAE,CAAC;QAC7F,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAEjF,IAAI,MAA6C,CAAC;QAClD,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAC/C,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,EACrF,GAAG,EAAE,CACH,IAAA,sBAAS,EACP;gBACE,KAAK,EAAE,cAAc;gBACrB,UAAU;gBACV,QAAQ,EAAE,KAAK,CAAC,cAAc,CAAC,QAAoC;gBACnE,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,MAAM;gBACnC,MAAM,EAAE;oBACN,WAAW,EAAE,KAAK,CAAC,SAAS,EAAE,WAAW;oBACzC,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,SAAS;oBACrC,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,IAAI;oBAC3B,KAAK,EAAE,KAAK,CAAC,cAAc,CAAC,KAAK;oBACjC,cAAc,EAAE,KAAK,CAAC,cAAc,CAAC,cAAc;oBACnD,SAAS,EAAE,KAAK,CAAC,cAAc,CAAC,SAAS;oBACzC,UAAU,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU;iBACxC;gBACD,UAAU,EAAE,KAAK,CAAC,KAAK,EAAE,UAAU;gBACnC,OAAO,EAAE;oBACP,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,cAAc,EAAE,UAAU,CAAC,cAAc;oBACzC,WAAW,EAAE,UAAU,CAAC,WAAW;oBACnC,WAAW,EAAE,UAAU,CAAC,WAAW;oBACnC,QAAQ,EAAE,UAAU,CAAC,QAAQ;oBAC7B,WAAW;oBACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,eAAe,EAAE,KAAK,CAAC,eAAe;oBACtC,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,OAAO,EAAE,UAAU,CAAC,OAAO;iBAC5B;gBACD,SAAS,EAAE;oBACT,EAAE,EAAE,WAAW;oBACf,SAAS,EAAE,KAAK,CAAC,SAAS;oBAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,IAAI;oBAChD,eAAe,EAAE,KAAK,CAAC,eAAe;oBACtC,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;oBAChC,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,IAAI;oBACpC,cAAc,EAAE,uBAAuB,CAAC,KAAK,CAAC,cAAc,CAAC;oBAC7D,cAAc,EAAE,KAAK,CAAC,cAAc;oBACpC,cAAc,EAAE,sBAAsB,CAAC,cAAc,CAAC;oBACtD,cAAc,EAAE,UAAU,CAAC,cAAc;oBACzC,WAAW,EAAE,UAAU,CAAC,WAAW;oBACnC,OAAO,EAAE,UAAU,CAAC,OAAO;oBAC3B,cAAc,EAAE,KAAK,CAAC,cAAc,IAAI,IAAI;iBAC7C;gBACD,gIAAgI;gBAChI,wHAAwH;gBACxH,aAAa,EAAE,kDAAqC;gBACpD,gBAAgB,EAAE,oBAAoB;aACvC,EACD;gBACE,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,eAAe;gBACf,iBAAiB,EAAE,CAAC,OAA+B,EAAE,EAAE;oBACrD,MAAM,aAAa,GACjB,OAAO,CAAC,aAAa,IAAI,OAAO,OAAO,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;oBACpG,OAAO,cAAc,CAAC,aAAa,EAAE;wBACnC,MAAM,EAAE,SAAS;wBACjB,UAAU,EAAE,OAAO,CAAC,GAAG;wBACvB,eAAe,EAAE,OAAO,CAAC,eAAe;wBACxC,oBAAoB,EAAE,aAAa,EAAE,oBAAoB,IAAI,IAAI;qBAClE,CAAC,CAAC;gBACL,CAAC;aACF,CACF,CACJ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,gCAAsB;gBAAE,MAAM,KAAK,CAAC;YACzD,MAAM,cAAc,CAAC,oBAAoB,EAAE;gBACzC,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO;aACzD,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,cAAc,CAAC,eAAe,EAAE;YACpC,MAAM,EAAE,WAAW;YACnB,SAAS,EAAE,MAAM,CAAC,UAAU;YAC5B,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW;YACrC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY;YACvC,YAAY,EAAE,MAAM,CAAC,YAAY;SAClC,CAAC,CAAC;QAEH,OAAO;YACL,WAAW;YACX,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,IAAI;YAC7C,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;YACnC,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,IAAI;YAC7C,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,yBAAyB,CAC7C,IAA+D,EAC/D,OAAe;IAEf,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,WAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,WAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEzG,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,eAAe,CAAC,2BAA2B,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,KAAK,CAAC;QAC3D,YAAY,EAAE,6BAA6B,CAAC,KAAK,CAAC,YAAY,CAAC;QAC/D,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,yBAAyB,EAAE,KAAK,CAAC,yBAAyB;QAC1D,0BAA0B,EAAE,KAAK,CAAC,0BAA0B;QAC5D,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,SAAgB,qBAAqB,CACnC,KAA4B,EAC5B,MAA+B;IAE/B,OAAO,IAAA,mCAAkB,EAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAsC;IACrE,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,eAAe,EAAE,KAAK,CAAC,cAAc;QACrC,UAAU,EAAE,KAAK,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAC5C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAClF,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAe;IACtC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAC/B,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,6BAA6B,CAAC,GAAY;IACjD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAI,GAA+B,CAAC,KAAK,CAAC;QACrD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnF,OAAO,EAAE,KAAK,EAAE,KAA6B,EAAE,CAAC;QAClD,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1D,OAAO,EAAE,GAAI,GAA+B,EAAE,CAAC;IACjD,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import type { DbClient } from '@proofhound/db';
|
|
2
|
-
import type
|
|
2
|
+
import { type RateLimiter } from '@proofhound/limiter';
|
|
3
3
|
import { type LLMCallLogger, type ModelConnectivityProbeResult } from '@proofhound/llm-client';
|
|
4
4
|
import type { ProbeJobPayload } from '@proofhound/orchestration-shared';
|
|
5
5
|
import type { LimiterKeyStrategy } from '../../server/common/contracts/limiter-key.strategy';
|
|
6
6
|
import type { QuotaPolicyHook } from '../../server/common/contracts/quota-policy.hook';
|
|
7
7
|
import type { RuntimeLimitsProvider } from '../../server/common/contracts/runtime-limits.provider';
|
|
8
|
+
import { type UsageMeteringHook } from '../../server/common/contracts/usage-metering.hook';
|
|
8
9
|
import type { ModelSecretResolver } from './model-secret';
|
|
9
10
|
export interface ProbeRunnerDependencies {
|
|
10
11
|
db: DbClient;
|
|
@@ -12,10 +13,16 @@ export interface ProbeRunnerDependencies {
|
|
|
12
13
|
limiterKeyStrategy: LimiterKeyStrategy;
|
|
13
14
|
quotaPolicy: QuotaPolicyHook;
|
|
14
15
|
runtimeLimitsProvider: RuntimeLimitsProvider;
|
|
16
|
+
usageMetering: UsageMeteringHook;
|
|
15
17
|
logger: LLMCallLogger;
|
|
16
18
|
modelSecretResolver: ModelSecretResolver;
|
|
17
19
|
}
|
|
18
|
-
export
|
|
20
|
+
export interface ProbeRunnerJobContext {
|
|
21
|
+
bullmqJobId: string;
|
|
22
|
+
bullmqQueue: string;
|
|
23
|
+
attempt: number;
|
|
24
|
+
}
|
|
25
|
+
export declare function createProbeRunner(deps: ProbeRunnerDependencies): (input: ProbeJobPayload, jobContext: ProbeRunnerJobContext) => Promise<ModelConnectivityProbeResult>;
|
|
19
26
|
export declare function listActiveModelIdsForProbe(db: DbClient, options?: {
|
|
20
27
|
limit?: number;
|
|
21
28
|
}): Promise<string[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"probe-runner.d.ts","sourceRoot":"","sources":["../../../src/worker/runners/probe-runner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"probe-runner.d.ts","sourceRoot":"","sources":["../../../src/worker/runners/probe-runner.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,EAA0B,KAAK,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,EAAyB,KAAK,aAAa,EAAE,KAAK,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AAEtH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oDAAoD,CAAC;AAC7F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iDAAiD,CAAC;AACvF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uDAAuD,CAAC;AACnG,OAAO,EAAwB,KAAK,iBAAiB,EAAE,MAAM,mDAAmD,CAAC;AAGjH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAE1D,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,QAAQ,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,WAAW,EAAE,eAAe,CAAC;IAC7B,qBAAqB,EAAE,qBAAqB,CAAC;IAC7C,aAAa,EAAE,iBAAiB,CAAC;IACjC,MAAM,EAAE,aAAa,CAAC;IACtB,mBAAmB,EAAE,mBAAmB,CAAC;CAC1C;AAED,MAAM,WAAW,qBAAqB;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,uBAAuB,IAE3D,OAAO,eAAe,EACtB,YAAY,qBAAqB,KAChC,OAAO,CAAC,4BAA4B,CAAC,CA8EzC;AAWD,wBAAsB,0BAA0B,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CASlH"}
|
|
@@ -4,14 +4,32 @@ exports.createProbeRunner = createProbeRunner;
|
|
|
4
4
|
exports.listActiveModelIdsForProbe = listActiveModelIdsForProbe;
|
|
5
5
|
const drizzle_orm_1 = require("drizzle-orm");
|
|
6
6
|
const db_1 = require("@proofhound/db");
|
|
7
|
+
const limiter_1 = require("@proofhound/limiter");
|
|
7
8
|
const llm_client_1 = require("@proofhound/llm-client");
|
|
8
9
|
const shared_1 = require("@proofhound/shared");
|
|
10
|
+
const usage_metering_hook_1 = require("../../server/common/contracts/usage-metering.hook");
|
|
9
11
|
const runtime_limits_1 = require("../../shared/llm/runtime-limits");
|
|
10
12
|
const llm_runner_1 = require("./llm-runner");
|
|
11
13
|
function createProbeRunner(deps) {
|
|
12
|
-
return async function runProbeJob(input) {
|
|
14
|
+
return async function runProbeJob(input, jobContext) {
|
|
13
15
|
const model = await (0, llm_runner_1.loadModelInvocationConfig)(deps, input.modelId);
|
|
14
16
|
const project = toProbeProjectContext(input);
|
|
17
|
+
const recordJobEvent = (eventType, payload) => (0, usage_metering_hook_1.safeRecordUsageEvent)(deps.usageMetering, {
|
|
18
|
+
idempotencyKey: `job:${jobContext.bullmqQueue}:${jobContext.bullmqJobId}:${jobContext.attempt}:${eventType}`,
|
|
19
|
+
dimension: 'job',
|
|
20
|
+
eventType,
|
|
21
|
+
projectId: project.projectId,
|
|
22
|
+
occurredAt: new Date(),
|
|
23
|
+
source: 'worker',
|
|
24
|
+
payload: {
|
|
25
|
+
queue: jobContext.bullmqQueue,
|
|
26
|
+
jobId: jobContext.bullmqJobId,
|
|
27
|
+
attempt: jobContext.attempt,
|
|
28
|
+
modelId: input.modelId,
|
|
29
|
+
source: 'probe',
|
|
30
|
+
...payload,
|
|
31
|
+
},
|
|
32
|
+
}, deps.logger);
|
|
15
33
|
const mergedLimits = await deps.runtimeLimitsProvider.mergeLlmLimits({
|
|
16
34
|
project,
|
|
17
35
|
modelId: input.modelId,
|
|
@@ -20,7 +38,33 @@ function createProbeRunner(deps) {
|
|
|
20
38
|
const effectiveModel = (0, runtime_limits_1.applyRuntimeLimits)(model, mergedLimits);
|
|
21
39
|
// Same key as the LLM runner so a probe shares the model's rate-limit counting space (§3.7).
|
|
22
40
|
const limiterKey = deps.limiterKeyStrategy.buildModelKey(project, input.modelId);
|
|
23
|
-
|
|
41
|
+
let result;
|
|
42
|
+
try {
|
|
43
|
+
result = await deps.quotaPolicy.withExecutionSlot({ project, source: 'probe', modelId: input.modelId, requestId: input.requestId }, () => (0, llm_client_1.testModelConnectivity)({ model: effectiveModel, limiterKey, requestId: input.requestId, timeoutMs: input.timeoutMs }, {
|
|
44
|
+
limiter: deps.limiter,
|
|
45
|
+
logger: deps.logger,
|
|
46
|
+
rethrowRateLimit: true,
|
|
47
|
+
onLimiterAcquired: (context) => recordJobEvent('job.started', {
|
|
48
|
+
status: 'started',
|
|
49
|
+
limiterKey: context.key,
|
|
50
|
+
estimatedTokens: context.estimatedTokens,
|
|
51
|
+
}),
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
if (error instanceof limiter_1.RateLimitExceededError)
|
|
56
|
+
throw error;
|
|
57
|
+
await recordJobEvent('job.failed', {
|
|
58
|
+
status: 'failed',
|
|
59
|
+
errorKind: error instanceof Error ? error.name : 'Error',
|
|
60
|
+
});
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
await recordJobEvent('job.completed', {
|
|
64
|
+
status: result.ok ? 'completed' : 'failed',
|
|
65
|
+
latencyMs: result.durationMs,
|
|
66
|
+
errorKind: result.ok ? null : (result.errorClass ?? 'probe_failed'),
|
|
67
|
+
});
|
|
24
68
|
await deps.db
|
|
25
69
|
.update(db_1.schema.models)
|
|
26
70
|
.set({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"probe-runner.js","sourceRoot":"","sources":["../../../src/worker/runners/probe-runner.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"probe-runner.js","sourceRoot":"","sources":["../../../src/worker/runners/probe-runner.ts"],"names":[],"mappings":";;AAgCA,8CAkFC;AAWD,gEASC;AAtID,6CAAiC;AAEjC,uCAAwC;AACxC,iDAA+E;AAC/E,uDAAsH;AACtH,+CAAgF;AAKhF,2FAAiH;AACjH,oEAAqE;AACrE,6CAAyD;AAoBzD,SAAgB,iBAAiB,CAAC,IAA6B;IAC7D,OAAO,KAAK,UAAU,WAAW,CAC/B,KAAsB,EACtB,UAAiC;QAEjC,MAAM,KAAK,GAAG,MAAM,IAAA,sCAAyB,EAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,cAAc,GAAG,CAAC,SAAiB,EAAE,OAAgC,EAAE,EAAE,CAC7E,IAAA,0CAAoB,EAClB,IAAI,CAAC,aAAa,EAClB;YACE,cAAc,EAAE,OAAO,UAAU,CAAC,WAAW,IAAI,UAAU,CAAC,WAAW,IAAI,UAAU,CAAC,OAAO,IAAI,SAAS,EAAE;YAC5G,SAAS,EAAE,KAAK;YAChB,SAAS;YACT,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE;gBACP,KAAK,EAAE,UAAU,CAAC,WAAW;gBAC7B,KAAK,EAAE,UAAU,CAAC,WAAW;gBAC7B,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,MAAM,EAAE,OAAO;gBACf,GAAG,OAAO;aACX;SACF,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;QACJ,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,cAAc,CAAC;YACnE,OAAO;YACP,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,OAAO;SAChB,CAAC,CAAC;QACH,MAAM,cAAc,GAAG,IAAA,mCAAkB,EAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QAC/D,6FAA6F;QAC7F,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACjF,IAAI,MAAoC,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAC/C,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,EAChF,GAAG,EAAE,CACH,IAAA,kCAAqB,EACnB,EAAE,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,EAC7F;gBACE,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,gBAAgB,EAAE,IAAI;gBACtB,iBAAiB,EAAE,CAAC,OAAO,EAAE,EAAE,CAC7B,cAAc,CAAC,aAAa,EAAE;oBAC5B,MAAM,EAAE,SAAS;oBACjB,UAAU,EAAE,OAAO,CAAC,GAAG;oBACvB,eAAe,EAAE,OAAO,CAAC,eAAe;iBACzC,CAAC;aACL,CACF,CACJ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,gCAAsB;gBAAE,MAAM,KAAK,CAAC;YACzD,MAAM,cAAc,CAAC,YAAY,EAAE;gBACjC,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO;aACzD,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,cAAc,CAAC,eAAe,EAAE;YACpC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YAC1C,SAAS,EAAE,MAAM,CAAC,UAAU;YAC5B,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,cAAc,CAAC;SACpE,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,EAAE;aACV,MAAM,CAAC,WAAM,CAAC,MAAM,CAAC;aACrB,GAAG,CAAC;YACH,YAAY,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACxC,cAAc,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,IAAI,iCAAiC,CAAC;YAC7F,SAAS,EAAE,IAAI,IAAI,EAAE;SACtB,CAAC;aACD,KAAK,CAAC,IAAA,gBAAE,EAAC,WAAM,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAE9C,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAsB;IACnD,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,OAAO,KAAK,CAAC,KAAK;YAChB,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE;YACrE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,8BAAqB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,8BAAqB,CAAC;AAChG,CAAC;AAEM,KAAK,UAAU,0BAA0B,CAAC,EAAY,EAAE,UAA8B,EAAE;IAC7F,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,EAAE;SAClB,MAAM,CAAC,EAAE,EAAE,EAAE,WAAM,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;SAChC,IAAI,CAAC,WAAM,CAAC,MAAM,CAAC;SACnB,KAAK,CAAC,IAAA,gBAAE,EAAC,WAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;SACvC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEhB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AACnC,CAAC"}
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import type { DbClient } from '@proofhound/db';
|
|
2
2
|
import type { LLMRunResultRecord, LLMRunResultWriter } from '@proofhound/llm-client';
|
|
3
3
|
import type { QuotaPolicyHook } from '../../server/common/contracts/quota-policy.hook';
|
|
4
|
+
import { type UsageMeteringHook } from '../../server/common/contracts/usage-metering.hook';
|
|
4
5
|
export declare class DrizzleRunResultWriter implements LLMRunResultWriter {
|
|
5
6
|
private readonly db;
|
|
6
7
|
private readonly quotaPolicy;
|
|
7
|
-
|
|
8
|
+
private readonly usageMetering?;
|
|
9
|
+
private readonly logger;
|
|
10
|
+
constructor(db: DbClient, quotaPolicy: QuotaPolicyHook, usageMetering?: UsageMeteringHook | undefined);
|
|
8
11
|
writeRunResult(record: LLMRunResultRecord): Promise<void>;
|
|
9
12
|
}
|
|
10
13
|
//# sourceMappingURL=run-result-writer.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-result-writer.d.ts","sourceRoot":"","sources":["../../../src/worker/runners/run-result-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"run-result-writer.d.ts","sourceRoot":"","sources":["../../../src/worker/runners/run-result-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAErF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iDAAiD,CAAC;AACvF,OAAO,EAAwB,KAAK,iBAAiB,EAAE,MAAM,mDAAmD,CAAC;AAMjH,qBAAa,sBAAuB,YAAW,kBAAkB;IAI7D,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IALjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmE;gBAGvE,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,eAAe,EAC5B,aAAa,CAAC,EAAE,iBAAiB,YAAA;IAG9C,cAAc,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;CA2FhE"}
|
|
@@ -3,14 +3,18 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.DrizzleRunResultWriter = void 0;
|
|
4
4
|
const node_buffer_1 = require("node:buffer");
|
|
5
5
|
const drizzle_orm_1 = require("drizzle-orm");
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
const logger_1 = require("@proofhound/logger");
|
|
7
|
+
const usage_metering_hook_1 = require("../../server/common/contracts/usage-metering.hook");
|
|
8
|
+
// ph_runs.run_results is monthly-partitioned by created_at, so UNIQUE(id) cannot live on that table.
|
|
9
|
+
// Reserve id in unpartitioned ph_runs.run_result_ids first, then insert into the partitioned fact table, ensuring:
|
|
8
10
|
// 1. worker stalled retries do not write duplicate rows
|
|
9
11
|
// 2. when the consumer writes the final error row in OnWorkerEvent('failed'), an already-landed success row is not overwritten
|
|
10
12
|
class DrizzleRunResultWriter {
|
|
11
|
-
constructor(db, quotaPolicy) {
|
|
13
|
+
constructor(db, quotaPolicy, usageMetering) {
|
|
12
14
|
this.db = db;
|
|
13
15
|
this.quotaPolicy = quotaPolicy;
|
|
16
|
+
this.usageMetering = usageMetering;
|
|
17
|
+
this.logger = (0, logger_1.createLogger)('worker.run-result-writer', { service: 'worker' });
|
|
14
18
|
}
|
|
15
19
|
async writeRunResult(record) {
|
|
16
20
|
const sampleId = record.sampleId ?? null;
|
|
@@ -39,17 +43,23 @@ class DrizzleRunResultWriter {
|
|
|
39
43
|
project: { projectId: record.projectId, source: 'local' },
|
|
40
44
|
source: 'run_result',
|
|
41
45
|
});
|
|
42
|
-
await this.db.execute((0, drizzle_orm_1.sql) `
|
|
46
|
+
const insertResult = await this.db.execute((0, drizzle_orm_1.sql) `
|
|
47
|
+
WITH reserved_run_result AS (
|
|
48
|
+
INSERT INTO ph_runs.run_result_ids (id)
|
|
49
|
+
VALUES (${record.id}::uuid)
|
|
50
|
+
ON CONFLICT (id) DO NOTHING
|
|
51
|
+
RETURNING id, created_at
|
|
52
|
+
)
|
|
43
53
|
INSERT INTO ph_runs.run_results (
|
|
44
54
|
id, project_id, source, source_id, release_variant_id, prompt_version_id, model_id,
|
|
45
55
|
sample_id, external_id, rendered_prompt, input_variables,
|
|
46
56
|
raw_response, parsed_output, decision_output, expected_output, is_correct, judgment_status,
|
|
47
57
|
status, error_class, error_message,
|
|
48
58
|
latency_ms, input_tokens, output_tokens, cost_estimate, attempt,
|
|
49
|
-
dbos_workflow_id, bullmq_job_id, round_index, webhook_token_id
|
|
59
|
+
dbos_workflow_id, bullmq_job_id, round_index, webhook_token_id, created_at
|
|
50
60
|
)
|
|
51
61
|
SELECT
|
|
52
|
-
|
|
62
|
+
reserved_run_result.id, ${record.projectId}::uuid, ${record.source},
|
|
53
63
|
${record.sourceId}::uuid, ${releaseVariantId}::uuid, ${record.promptVersionId}::uuid, ${record.modelId}::uuid,
|
|
54
64
|
${sampleId}::uuid, ${externalId},
|
|
55
65
|
${JSON.stringify(record.renderedPrompt)}::jsonb,
|
|
@@ -60,11 +70,36 @@ class DrizzleRunResultWriter {
|
|
|
60
70
|
${record.status}, ${errorClass}, ${errorMessage},
|
|
61
71
|
${latencyMs}, ${inputTokens}, ${outputTokens},
|
|
62
72
|
${costEstimate}, ${attempt},
|
|
63
|
-
${dbosWorkflowId}, ${bullmqJobId}, ${roundIndex}, ${webhookTokenId}::uuid
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
73
|
+
${dbosWorkflowId}, ${bullmqJobId}, ${roundIndex}, ${webhookTokenId}::uuid,
|
|
74
|
+
reserved_run_result.created_at
|
|
75
|
+
FROM reserved_run_result
|
|
76
|
+
RETURNING id, created_at
|
|
67
77
|
`);
|
|
78
|
+
const insertedRows = unwrapRows(insertResult);
|
|
79
|
+
if (insertedRows.length > 0 && this.usageMetering) {
|
|
80
|
+
const occurredAt = coerceDate(insertedRows[0]?.created_at);
|
|
81
|
+
await (0, usage_metering_hook_1.safeRecordUsageEvent)(this.usageMetering, {
|
|
82
|
+
idempotencyKey: `run_result:${record.id}:created`,
|
|
83
|
+
dimension: 'run_result',
|
|
84
|
+
eventType: 'run_result.created',
|
|
85
|
+
projectId: record.projectId,
|
|
86
|
+
occurredAt,
|
|
87
|
+
source: 'worker',
|
|
88
|
+
payload: {
|
|
89
|
+
runResultId: record.id,
|
|
90
|
+
source: record.source,
|
|
91
|
+
sourceId: record.sourceId,
|
|
92
|
+
promptVersionId: record.promptVersionId,
|
|
93
|
+
modelId: record.modelId,
|
|
94
|
+
status: record.status,
|
|
95
|
+
inputTokens,
|
|
96
|
+
outputTokens,
|
|
97
|
+
costEstimate,
|
|
98
|
+
latencyMs,
|
|
99
|
+
createdAt: occurredAt.toISOString(),
|
|
100
|
+
},
|
|
101
|
+
}, this.logger);
|
|
102
|
+
}
|
|
68
103
|
}
|
|
69
104
|
}
|
|
70
105
|
exports.DrizzleRunResultWriter = DrizzleRunResultWriter;
|
|
@@ -81,4 +116,19 @@ function estimateRunResultBytes(record) {
|
|
|
81
116
|
function utf8Bytes(value) {
|
|
82
117
|
return node_buffer_1.Buffer.byteLength(typeof value === 'string' ? value : JSON.stringify(value ?? null), 'utf8');
|
|
83
118
|
}
|
|
119
|
+
function unwrapRows(result) {
|
|
120
|
+
if (Array.isArray(result))
|
|
121
|
+
return result;
|
|
122
|
+
if (result && typeof result === 'object' && 'rows' in result) {
|
|
123
|
+
return (result.rows ?? []);
|
|
124
|
+
}
|
|
125
|
+
return [];
|
|
126
|
+
}
|
|
127
|
+
function coerceDate(value) {
|
|
128
|
+
if (value instanceof Date)
|
|
129
|
+
return value;
|
|
130
|
+
if (value)
|
|
131
|
+
return new Date(value);
|
|
132
|
+
return new Date();
|
|
133
|
+
}
|
|
84
134
|
//# sourceMappingURL=run-result-writer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-result-writer.js","sourceRoot":"","sources":["../../../src/worker/runners/run-result-writer.ts"],"names":[],"mappings":";;;AAAA,6CAAqC;AACrC,6CAAkC;
|
|
1
|
+
{"version":3,"file":"run-result-writer.js","sourceRoot":"","sources":["../../../src/worker/runners/run-result-writer.ts"],"names":[],"mappings":";;;AAAA,6CAAqC;AACrC,6CAAkC;AAGlC,+CAAkD;AAElD,2FAAiH;AAEjH,qGAAqG;AACrG,mHAAmH;AACnH,yDAAyD;AACzD,gIAAgI;AAChI,MAAa,sBAAsB;IAGjC,YACmB,EAAY,EACZ,WAA4B,EAC5B,aAAiC;QAFjC,OAAE,GAAF,EAAE,CAAU;QACZ,gBAAW,GAAX,WAAW,CAAiB;QAC5B,kBAAa,GAAb,aAAa,CAAoB;QALnC,WAAM,GAAG,IAAA,qBAAY,EAAC,0BAA0B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;IAMvF,CAAC;IAEJ,KAAK,CAAC,cAAc,CAAC,MAA0B;QAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;QACzC,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;QAC7C,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QACrD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC/C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC;QACjD,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC;QAC3C,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QACrD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC/C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC;QACjD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC;QACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;QACpC,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QACrD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;QAC7C,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC;QACzD,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QACrD,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC;YACpC,KAAK,EAAE,sBAAsB,CAAC,MAAM,CAAC;YACrC,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;YACzD,MAAM,EAAE,YAAY;SACrB,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAA6C,IAAA,iBAAG,EAAA;;;kBAG5E,MAAM,CAAC,EAAE;;;;;;;;;;;;;kCAaO,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,MAAM;UAChE,MAAM,CAAC,QAAQ,WAAW,gBAAgB,WAAW,MAAM,CAAC,eAAe,WAAW,MAAM,CAAC,OAAO;UACpG,QAAQ,WAAW,UAAU;UAC7B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC;UACrC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC;UAC9B,WAAW,KAAK,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC;UAC5C,cAAc,KAAK,cAAc;UACjC,SAAS,KAAK,cAAc;UAC5B,MAAM,CAAC,MAAM,KAAK,UAAU,KAAK,YAAY;UAC7C,SAAS,KAAK,WAAW,KAAK,YAAY;UAC1C,YAAY,KAAK,OAAO;UACxB,cAAc,KAAK,WAAW,KAAK,UAAU,KAAK,cAAc;;;;KAIrE,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,UAAU,CAA6C,YAAY,CAAC,CAAC;QAC1F,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAClD,MAAM,UAAU,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YAC3D,MAAM,IAAA,0CAAoB,EACxB,IAAI,CAAC,aAAa,EAClB;gBACE,cAAc,EAAE,cAAc,MAAM,CAAC,EAAE,UAAU;gBACjD,SAAS,EAAE,YAAY;gBACvB,SAAS,EAAE,oBAAoB;gBAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,UAAU;gBACV,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,EAAE;oBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,eAAe,EAAE,MAAM,CAAC,eAAe;oBACvC,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,WAAW;oBACX,YAAY;oBACZ,YAAY;oBACZ,SAAS;oBACT,SAAS,EAAE,UAAU,CAAC,WAAW,EAAE;iBACpC;aACF,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AApGD,wDAoGC;AAED,SAAS,sBAAsB,CAAC,MAA0B;IACxD,OAAO,CACL,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC;QAChC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC;QAChC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC;QAC7B,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC;QAC9B,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC;QAChC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC;QAChC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC;QAC5B,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,oBAAM,CAAC,UAAU,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;AACtG,CAAC;AAED,SAAS,UAAU,CAAc,MAAe;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,MAAa,CAAC;IAChD,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAK,MAAkC,EAAE,CAAC;QAC1F,OAAO,CAAE,MAAyB,CAAC,IAAI,IAAI,EAAE,CAAQ,CAAC;IACxD,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CAAC,KAAgC;IAClD,IAAI,KAAK,YAAY,IAAI;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,KAAK;QAAE,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,IAAI,EAAE,CAAC;AACpB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@proofhound/core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"description": "ProofHound reusable backend runtime: server / webhook / worker NestJS modules, contracts, local defaults, Services, Repositories",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -36,17 +36,17 @@
|
|
|
36
36
|
"rxjs": "^7.8.2",
|
|
37
37
|
"sharp": "^0.34.5",
|
|
38
38
|
"zod": "^4.4.3",
|
|
39
|
-
"@proofhound/
|
|
40
|
-
"@proofhound/
|
|
41
|
-
"@proofhound/
|
|
42
|
-
"@proofhound/
|
|
43
|
-
"@proofhound/limiter": "0.1.
|
|
44
|
-
"@proofhound/llm-client": "0.1.
|
|
45
|
-
"@proofhound/
|
|
46
|
-
"@proofhound/
|
|
47
|
-
"@proofhound/
|
|
48
|
-
"@proofhound/shared": "0.1.
|
|
49
|
-
"@proofhound/
|
|
39
|
+
"@proofhound/db": "0.1.8",
|
|
40
|
+
"@proofhound/connector-client": "0.1.8",
|
|
41
|
+
"@proofhound/crypto": "0.1.8",
|
|
42
|
+
"@proofhound/judgment": "0.1.8",
|
|
43
|
+
"@proofhound/limiter": "0.1.8",
|
|
44
|
+
"@proofhound/llm-client": "0.1.8",
|
|
45
|
+
"@proofhound/logger": "0.1.8",
|
|
46
|
+
"@proofhound/metrics": "0.1.8",
|
|
47
|
+
"@proofhound/optimization-strategy": "0.1.8",
|
|
48
|
+
"@proofhound/shared": "0.1.8",
|
|
49
|
+
"@proofhound/orchestration-shared": "0.1.8"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@nestjs/testing": "^11.1.21",
|
|
@@ -80,7 +80,7 @@
|
|
|
80
80
|
},
|
|
81
81
|
"scripts": {
|
|
82
82
|
"build": "rm -rf dist && tsc -p tsconfig.build.json",
|
|
83
|
-
"typecheck": "tsc --noEmit",
|
|
83
|
+
"typecheck": "tsc --noEmit --incremental false",
|
|
84
84
|
"lint": "eslint \"src/**/*.ts\"",
|
|
85
85
|
"test": "vitest run",
|
|
86
86
|
"test:unit": "vitest run",
|