@pikku/mongodb 0.12.18 → 0.12.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +134 -0
- package/dist/src/index.d.ts +2 -2
- package/dist/src/mongodb-agent-run-service.d.ts +1 -2
- package/dist/src/mongodb-agent-storage-service.d.ts +2 -2
- package/dist/src/mongodb-channel-store.d.ts +1 -1
- package/dist/src/mongodb-deployment-service.d.ts +3 -2
- package/dist/src/mongodb-deployment-service.js +2 -2
- package/dist/src/mongodb-eventhub-store.d.ts +1 -1
- package/dist/src/mongodb-eventhub-store.js +1 -1
- package/dist/src/mongodb-secret-service.d.ts +1 -1
- package/dist/src/mongodb-secret-service.js +2 -2
- package/dist/src/mongodb-session-store.d.ts +1 -1
- package/dist/src/mongodb-workflow-run-service.d.ts +1 -1
- package/dist/src/mongodb-workflow-service.d.ts +24 -2
- package/dist/src/mongodb-workflow-service.js +53 -1
- package/package.json +6 -6
- package/src/index.ts +2 -5
- package/src/mongodb-agent-run-service.ts +3 -2
- package/src/mongodb-agent-storage-service.ts +7 -7
- package/src/mongodb-channel-store.ts +1 -1
- package/src/mongodb-deployment-service.ts +5 -4
- package/src/mongodb-eventhub-store.ts +1 -1
- package/src/mongodb-secret-service.ts +6 -5
- package/src/mongodb-services.test.ts +1 -1
- package/src/mongodb-session-store.ts +1 -1
- package/src/mongodb-workflow-run-service.ts +1 -1
- package/src/mongodb-workflow-service.test.ts +117 -0
- package/src/mongodb-workflow-service.ts +89 -3
- package/dist/src/mongodb-ai-storage-service.d.ts +0 -45
- package/dist/src/mongodb-ai-storage-service.js +0 -438
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,139 @@
|
|
|
1
1
|
# @pikku/mongodb
|
|
2
2
|
|
|
3
|
+
## 0.12.20
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 3c0012c: Gate console agent-thread reads and deletes on thread ownership, claim MongoDB workflow steps atomically, and reach the deployment fallback from `rpcWithWire`
|
|
8
|
+
|
|
9
|
+
`getAgentThreadMessages` and `deleteAgentThread` in the console addon took a
|
|
10
|
+
caller-supplied `threadId` straight to storage, while their siblings
|
|
11
|
+
`getAgentThreads` and `getAgentThreadRuns` already filtered to what the session
|
|
12
|
+
owns. Both now carry an `isThreadOwner` permission: an admin reaches any thread,
|
|
13
|
+
everyone else only their own, and a missing thread is refused rather than 404'd
|
|
14
|
+
so it is indistinguishable from someone else's.
|
|
15
|
+
|
|
16
|
+
`MongoDBWorkflowService` claimed a step by reading its status and then writing
|
|
17
|
+
it, under a `withStepLock` that is a pass-through — so two dispatches racing for
|
|
18
|
+
the same step could both proceed and run a side-effecting step twice. The claim
|
|
19
|
+
is now a single status-guarded update, atomic on one document.
|
|
20
|
+
|
|
21
|
+
`rpcWithWire` threw `RPCNotFoundError` for any unresolved namespaced call
|
|
22
|
+
instead of falling through to the deployment service, so a namespaced RPC that
|
|
23
|
+
`rpc()` would have dispatched remotely failed when called with an explicit wire.
|
|
24
|
+
|
|
25
|
+
- Updated dependencies [3c0012c]
|
|
26
|
+
- Updated dependencies [05e47cf]
|
|
27
|
+
- Updated dependencies [cfd364a]
|
|
28
|
+
- Updated dependencies [05e47cf]
|
|
29
|
+
- Updated dependencies [05e47cf]
|
|
30
|
+
- Updated dependencies [05e47cf]
|
|
31
|
+
- @pikku/core@0.12.90
|
|
32
|
+
|
|
33
|
+
## 0.12.19
|
|
34
|
+
|
|
35
|
+
### Patch Changes
|
|
36
|
+
|
|
37
|
+
- 266e3bc: One door per name: `@pikku/core/ecosystem/*` and the package root are gone
|
|
38
|
+
|
|
39
|
+
`@pikku/core` published every module twice. `ecosystem/http` re-exported
|
|
40
|
+
`./http`, `ecosystem/services` re-exported `./services`, and a name was
|
|
41
|
+
reachable through either — so every addition had to be made in two places, and a
|
|
42
|
+
consumer's import said nothing about what it actually used. The package root was
|
|
43
|
+
the same problem at a larger scale: a single barrel of 206 names that no bundler
|
|
44
|
+
could take apart, and the one specifier that revealed nothing at all.
|
|
45
|
+
|
|
46
|
+
Both are deleted. Every name now lives on the subpath that owns it, and every
|
|
47
|
+
import carries that subpath — `@pikku/core/http`, `@pikku/core/services`,
|
|
48
|
+
`@pikku/core/errors`, `@pikku/core/types`.
|
|
49
|
+
|
|
50
|
+
Deleting the facades meant the raw subpaths had to become a superset of them,
|
|
51
|
+
which they were not: the facade tree had accumulated 25 names with no raw home
|
|
52
|
+
and about 26 more filed under a different area than the module they came from.
|
|
53
|
+
Those names moved to the area that owns them, and three areas were published as
|
|
54
|
+
new entry points rather than left on a root that is going away — `./types` (the
|
|
55
|
+
shared type surface, the largest single destination), `./state` and
|
|
56
|
+
`./classification`.
|
|
57
|
+
|
|
58
|
+
`./classification` is one door onto one subject: what a value is and how it must
|
|
59
|
+
be handled. Its three halves would each have been an entry point — the brands
|
|
60
|
+
and manifest types, the stored-form helpers (`hashToken`, `unsafeAsSealed` and
|
|
61
|
+
friends), and `SecretValue` — split by whether a name happens to be a type or a
|
|
62
|
+
value, which is the same defect as the facades. The duration and versioned-id
|
|
63
|
+
helpers went to `./utils`, which already published, and `PikkuRequest` went to
|
|
64
|
+
`./function`: it is the transport-agnostic request base, not an HTTP one — HTTP
|
|
65
|
+
has `PikkuHTTPAbstractRequest`, and the only thing outside core that extends
|
|
66
|
+
`PikkuRequest` is Azure's timer request.
|
|
67
|
+
|
|
68
|
+
`./types` inherited the root barrel's habit before it inherited its contents, so
|
|
69
|
+
the names with an owner elsewhere were moved off it. The middleware types and the
|
|
70
|
+
five middleware factories — `pikkuMiddleware`, `pikkuMiddlewareFactory`,
|
|
71
|
+
`pikkuChannelMiddleware`, `pikkuChannelMiddlewareFactory` and
|
|
72
|
+
`pikkuAgentMiddleware`, runtime values on a types entry point — are now
|
|
73
|
+
`@pikku/core/middleware`; the function meta types are `@pikku/core/function`;
|
|
74
|
+
`SerializedError` is `@pikku/core/errors`; and the generic TypeScript helpers
|
|
75
|
+
(`MakeRequired`, `PickRequired`, `PickOptional`, `RequireAtLeastOne`,
|
|
76
|
+
`JSONPrimitive`, `JSONValue`) are `@pikku/core/utils`. What is left on `./types`
|
|
77
|
+
is the vocabulary the wirings share, which no single module owns.
|
|
78
|
+
|
|
79
|
+
`pikku` was itself a root barrel — `export * from '@pikku/core'` — and
|
|
80
|
+
now exports only the services it bundles.
|
|
81
|
+
|
|
82
|
+
One module survives at the old specifier, and only for the bootstrap:
|
|
83
|
+
`packages/cli` is generated by the _published_ CLI pinned in its `build.sh`, and
|
|
84
|
+
that CLI still writes a bare `@pikku/core` into the files it generates for the
|
|
85
|
+
CLI itself. `bootstrap-compat/root.ts` carries the eight types it names, a test
|
|
86
|
+
in core fails if that list grows, and it goes when the pin moves to a CLI
|
|
87
|
+
released from this branch. The adapter names the pinned CLI reaches for —
|
|
88
|
+
`pikkuState` and `CreateWireServices` — are rewritten to `@pikku/core/state` and
|
|
89
|
+
`@pikku/core/types` by the same `build.sh` patch pass, so no second shim is
|
|
90
|
+
needed for them.
|
|
91
|
+
|
|
92
|
+
A guard test keeps the root shut: it parses imports and rejects a bare
|
|
93
|
+
`@pikku/core` rather than grepping for it, because several tests hold a user's
|
|
94
|
+
file as a template literal, where `import … from '@pikku/core'` is fixture text
|
|
95
|
+
rather than an import this repo makes.
|
|
96
|
+
|
|
97
|
+
An agent scaffold a project generated under an older CLI is refreshed rather
|
|
98
|
+
than left to fail: `pikku all` already deleted one importing an entry point
|
|
99
|
+
`@pikku/core` no longer publishes, and the `#pikku` hub joins that list.
|
|
100
|
+
Without it a project that scaffolds the agent endpoint but
|
|
101
|
+
declares no agents keeps the old file forever — the generator that would rewrite
|
|
102
|
+
it only runs when agents exist, and the file being present is what stops it
|
|
103
|
+
being regenerated as missing.
|
|
104
|
+
|
|
105
|
+
`pikku new addon` also wrote a tsconfig `paths` map naming only the deleted hub.
|
|
106
|
+
An addon's `imports` map points into `dist`, so `paths` is what resolves
|
|
107
|
+
`#pikku/<leaf>` for the addon's own source build — it now names the two leaf
|
|
108
|
+
patterns, in both the addon and its test harness.
|
|
109
|
+
|
|
110
|
+
- 786dae5: Bump every dependency whose latest release is a major across the monorepo, and
|
|
111
|
+
port the code the majors broke: `cookie` 2's `parseCookie`/`stringifySetCookie`
|
|
112
|
+
API in `@pikku/core` and the three runtime HTTP adapters, and assistant-ui 0.15's
|
|
113
|
+
store client in `@pikku/assistant-ui`.
|
|
114
|
+
- 6eef0a0: Bump every dependency to its latest compatible minor/patch across the monorepo.
|
|
115
|
+
- Updated dependencies [7722ceb]
|
|
116
|
+
- Updated dependencies [375c1ff]
|
|
117
|
+
- Updated dependencies [02a70cd]
|
|
118
|
+
- Updated dependencies [aeef159]
|
|
119
|
+
- Updated dependencies [a281de6]
|
|
120
|
+
- Updated dependencies [266e3bc]
|
|
121
|
+
- Updated dependencies [02a70cd]
|
|
122
|
+
- Updated dependencies [786dae5]
|
|
123
|
+
- Updated dependencies [6eef0a0]
|
|
124
|
+
- Updated dependencies [3561d67]
|
|
125
|
+
- Updated dependencies [a91c433]
|
|
126
|
+
- Updated dependencies [02a70cd]
|
|
127
|
+
- Updated dependencies [9537f74]
|
|
128
|
+
- Updated dependencies [2b57ca8]
|
|
129
|
+
- Updated dependencies [266e3bc]
|
|
130
|
+
- Updated dependencies [9fce0f1]
|
|
131
|
+
- Updated dependencies [83683a0]
|
|
132
|
+
- Updated dependencies [456c88b]
|
|
133
|
+
- Updated dependencies [456c88b]
|
|
134
|
+
- Updated dependencies [c127273]
|
|
135
|
+
- @pikku/core@0.12.85
|
|
136
|
+
|
|
3
137
|
## 0.12.18
|
|
4
138
|
|
|
5
139
|
### Patch Changes
|
package/dist/src/index.d.ts
CHANGED
|
@@ -7,6 +7,6 @@ export { MongoDBDeploymentService } from './mongodb-deployment-service.js';
|
|
|
7
7
|
export { MongoDBAgentStorageService } from './mongodb-agent-storage-service.js';
|
|
8
8
|
export { MongoDBAgentRunService } from './mongodb-agent-run-service.js';
|
|
9
9
|
export { MongoDBSecretService } from './mongodb-secret-service.js';
|
|
10
|
-
export type { WorkflowRunService } from '@pikku/core/
|
|
11
|
-
export type { AgentRunService, AgentRunRow
|
|
10
|
+
export type { WorkflowRunService } from '@pikku/core/workflow';
|
|
11
|
+
export type { AgentRunService, AgentRunRow } from '@pikku/core/agent';
|
|
12
12
|
export { MongoDBSessionStore } from './mongodb-session-store.js';
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type { AgentThread, AgentMessage } from '@pikku/core/agent';
|
|
2
|
-
import type { AgentRunRow, AgentRunService } from '@pikku/core/ecosystem/agent';
|
|
1
|
+
import type { AgentThread, AgentMessage, AgentRunRow, AgentRunService } from '@pikku/core/agent';
|
|
3
2
|
import type { Db } from 'mongodb';
|
|
4
3
|
export declare class MongoDBAgentRunService implements AgentRunService {
|
|
5
4
|
private threads;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { AgentThread, AgentMessage, AgentRunState } from '@pikku/core/agent';
|
|
1
|
+
import type { AgentMessage, AgentRunState, AgentThread } from '@pikku/core/agent';
|
|
3
2
|
import type { AgentRunScore } from '@pikku/core/agent-scorer';
|
|
3
|
+
import type { AgentRunStateService, AgentStorageService, CreateRunInput, SaveScoreInput } from '@pikku/core/services';
|
|
4
4
|
import type { Db } from 'mongodb';
|
|
5
5
|
export declare class MongoDBAgentStorageService implements AgentStorageService, AgentRunStateService {
|
|
6
6
|
private db;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { Channel } from '@pikku/core/channel';
|
|
2
1
|
import { ChannelStore } from '@pikku/core/channel';
|
|
2
|
+
import type { Channel } from '@pikku/core/channel';
|
|
3
3
|
import type { Db } from 'mongodb';
|
|
4
4
|
export declare class MongoDBChannelStore extends ChannelStore {
|
|
5
5
|
private db;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import type { DeploymentService, DeploymentServiceConfig, DeploymentConfig } from '@pikku/core/
|
|
2
|
-
import type { JWTService
|
|
1
|
+
import type { DeploymentService, DeploymentServiceConfig, DeploymentConfig } from '@pikku/core/services';
|
|
2
|
+
import type { JWTService } from '@pikku/core/services';
|
|
3
|
+
import type { SecretService } from '@pikku/core/services';
|
|
3
4
|
import type { Db } from 'mongodb';
|
|
4
5
|
export declare class MongoDBDeploymentService implements DeploymentService {
|
|
5
6
|
private db;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { buildRemoteHeaders } from '@pikku/core/
|
|
2
|
-
import { getAllFunctionNames } from '@pikku/core/
|
|
1
|
+
import { buildRemoteHeaders } from '@pikku/core/remote';
|
|
2
|
+
import { getAllFunctionNames } from '@pikku/core/function';
|
|
3
3
|
export class MongoDBDeploymentService {
|
|
4
4
|
db;
|
|
5
5
|
jwt;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type SecretValue } from '@pikku/core';
|
|
2
1
|
import type { SecretService, SecretValues } from '@pikku/core/services';
|
|
2
|
+
import type { SecretValue } from '@pikku/core/classification';
|
|
3
3
|
import type { Db } from 'mongodb';
|
|
4
4
|
export interface MongoDBSecretServiceConfig {
|
|
5
5
|
key: string;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { deriveKEK, envelopeDecrypt, envelopeEncrypt, envelopeRewrap, generateKEKSalt, } from '@pikku/core/crypto-utils';
|
|
2
|
+
import { createSecretValue, isSecretValue } from '@pikku/core/classification';
|
|
3
3
|
export class MongoDBSecretService {
|
|
4
4
|
db;
|
|
5
5
|
initialized = false;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { CoreUserSession } from '@pikku/core';
|
|
2
1
|
import type { SessionStore } from '@pikku/core/services';
|
|
2
|
+
import type { CoreUserSession } from '@pikku/core/types';
|
|
3
3
|
import type { Db } from 'mongodb';
|
|
4
4
|
export declare class MongoDBSessionStore implements SessionStore {
|
|
5
5
|
private db;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { WorkflowRun, StepState, WorkflowRunService } from '@pikku/core/
|
|
1
|
+
import type { WorkflowRun, StepState, WorkflowRunService } from '@pikku/core/workflow';
|
|
2
2
|
import type { Db } from 'mongodb';
|
|
3
3
|
export declare class MongoDBWorkflowRunService implements WorkflowRunService {
|
|
4
4
|
private runs;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { SerializedError } from '@pikku/core/
|
|
1
|
+
import type { SerializedError } from '@pikku/core/errors';
|
|
2
2
|
import { PikkuWorkflowService } from '@pikku/core/workflow';
|
|
3
|
-
import type { WorkflowPlannedStep, WorkflowQueueOptions, WorkflowRun, WorkflowRunWire, StepState, StepStatus, WorkflowStatus, WorkflowVersionStatus } from '@pikku/core/
|
|
3
|
+
import type { WorkflowPlannedStep, WorkflowQueueOptions, WorkflowRun, WorkflowRunWire, StepState, StepStatus, WorkflowStatus, WorkflowVersionStatus } from '@pikku/core/workflow';
|
|
4
4
|
import type { Db } from 'mongodb';
|
|
5
5
|
export declare class MongoDBWorkflowService extends PikkuWorkflowService {
|
|
6
6
|
private initialized;
|
|
@@ -44,7 +44,29 @@ export declare class MongoDBWorkflowService extends PikkuWorkflowService {
|
|
|
44
44
|
protected setStepErrorImpl(stepId: string, error: Error): Promise<void>;
|
|
45
45
|
protected createRetryAttemptImpl(stepId: string, status: 'pending' | 'running'): Promise<StepState>;
|
|
46
46
|
withRunLock<T>(id: string, fn: () => Promise<T>): Promise<T>;
|
|
47
|
+
/**
|
|
48
|
+
* A pass-through: MongoDB has no advisory-lock primitive to build one on.
|
|
49
|
+
* The one decision that must exclude — claiming a step to execute it — is
|
|
50
|
+
* made by `claimStepForExecution` below as a single conditional write, which
|
|
51
|
+
* needs no lock to be exclusive.
|
|
52
|
+
*/
|
|
47
53
|
withStepLock<T>(_runId: string, _stepName: string, fn: () => Promise<T>): Promise<T>;
|
|
54
|
+
/**
|
|
55
|
+
* Claim the step with a status-guarded update and read the modified count:
|
|
56
|
+
* a single-document update is atomic in MongoDB, so two dispatches racing for
|
|
57
|
+
* the same step cannot both proceed.
|
|
58
|
+
*
|
|
59
|
+
* This replaces the read-then-write the base engine does under
|
|
60
|
+
* `withStepLock`, which excludes nothing here — the lock above is a
|
|
61
|
+
* pass-through. The winner then goes through the ordinary transition methods,
|
|
62
|
+
* so history rows see exactly what they saw before.
|
|
63
|
+
*/
|
|
64
|
+
protected claimStepForExecution(runId: string, stepName: string, rpcName: string): Promise<StepState | null>;
|
|
65
|
+
/**
|
|
66
|
+
* Move a step to `running` only if it is still in one of `from`, reporting
|
|
67
|
+
* whether this caller is the one that moved it.
|
|
68
|
+
*/
|
|
69
|
+
private claimStepStatus;
|
|
48
70
|
getCompletedGraphState(runId: string): Promise<{
|
|
49
71
|
completedNodeIds: string[];
|
|
50
72
|
failedNodeIds: string[];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { PikkuWorkflowService } from '@pikku/core/workflow';
|
|
1
|
+
import { PikkuWorkflowService, WorkflowStepFunctionMismatchError, } from '@pikku/core/workflow';
|
|
2
2
|
import { MongoDBWorkflowRunService } from './mongodb-workflow-run-service.js';
|
|
3
3
|
export class MongoDBWorkflowService extends PikkuWorkflowService {
|
|
4
4
|
initialized = false;
|
|
@@ -254,9 +254,61 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
|
|
|
254
254
|
async withRunLock(id, fn) {
|
|
255
255
|
return fn();
|
|
256
256
|
}
|
|
257
|
+
/**
|
|
258
|
+
* A pass-through: MongoDB has no advisory-lock primitive to build one on.
|
|
259
|
+
* The one decision that must exclude — claiming a step to execute it — is
|
|
260
|
+
* made by `claimStepForExecution` below as a single conditional write, which
|
|
261
|
+
* needs no lock to be exclusive.
|
|
262
|
+
*/
|
|
257
263
|
async withStepLock(_runId, _stepName, fn) {
|
|
258
264
|
return fn();
|
|
259
265
|
}
|
|
266
|
+
/**
|
|
267
|
+
* Claim the step with a status-guarded update and read the modified count:
|
|
268
|
+
* a single-document update is atomic in MongoDB, so two dispatches racing for
|
|
269
|
+
* the same step cannot both proceed.
|
|
270
|
+
*
|
|
271
|
+
* This replaces the read-then-write the base engine does under
|
|
272
|
+
* `withStepLock`, which excludes nothing here — the lock above is a
|
|
273
|
+
* pass-through. The winner then goes through the ordinary transition methods,
|
|
274
|
+
* so history rows see exactly what they saw before.
|
|
275
|
+
*/
|
|
276
|
+
async claimStepForExecution(runId, stepName, rpcName) {
|
|
277
|
+
const stepState = await this.getStepState(runId, stepName);
|
|
278
|
+
// knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
|
|
279
|
+
if (stepState.rpcName !== undefined &&
|
|
280
|
+
stepState.rpcName !== (rpcName ?? null)) {
|
|
281
|
+
throw new WorkflowStepFunctionMismatchError(runId, stepName);
|
|
282
|
+
}
|
|
283
|
+
if (stepState.status === 'succeeded' || stepState.status === 'running') {
|
|
284
|
+
return null;
|
|
285
|
+
}
|
|
286
|
+
if (stepState.status === 'failed') {
|
|
287
|
+
if (!(await this.claimStepStatus(stepState.stepId, ['failed']))) {
|
|
288
|
+
return null;
|
|
289
|
+
}
|
|
290
|
+
return this.createRetryAttempt(stepState.stepId, 'running');
|
|
291
|
+
}
|
|
292
|
+
if (stepState.status === 'pending' || stepState.status === 'scheduled') {
|
|
293
|
+
if (!(await this.claimStepStatus(stepState.stepId, [
|
|
294
|
+
'pending',
|
|
295
|
+
'scheduled',
|
|
296
|
+
]))) {
|
|
297
|
+
return null;
|
|
298
|
+
}
|
|
299
|
+
await this.setStepRunning(stepState.stepId);
|
|
300
|
+
return { ...stepState, status: 'running' };
|
|
301
|
+
}
|
|
302
|
+
return stepState;
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* Move a step to `running` only if it is still in one of `from`, reporting
|
|
306
|
+
* whether this caller is the one that moved it.
|
|
307
|
+
*/
|
|
308
|
+
async claimStepStatus(stepId, from) {
|
|
309
|
+
const claimed = await this.steps.updateOne({ _id: stepId, status: { $in: from } }, { $set: { status: 'running', updatedAt: new Date() } });
|
|
310
|
+
return claimed.modifiedCount > 0;
|
|
311
|
+
}
|
|
260
312
|
async getCompletedGraphState(runId) {
|
|
261
313
|
const results = await this.steps
|
|
262
314
|
.find({
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/mongodb",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.20",
|
|
4
4
|
"description": "MongoDB services for Pikku — workflow state, agent runs, secrets and channel stores",
|
|
5
5
|
"author": "yasser.fadl@gmail.com",
|
|
6
6
|
"license": "MIT",
|
|
@@ -18,17 +18,17 @@
|
|
|
18
18
|
"prepublishOnly": "yarn build"
|
|
19
19
|
},
|
|
20
20
|
"peerDependencies": {
|
|
21
|
-
"@pikku/core": "^0.12.
|
|
21
|
+
"@pikku/core": "^0.12.90"
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"mongodb": "^
|
|
24
|
+
"mongodb": "^7.5.0"
|
|
25
25
|
},
|
|
26
26
|
"devDependencies": {
|
|
27
|
-
"mongodb": "^
|
|
28
|
-
"mongodb-memory-server": "^
|
|
27
|
+
"mongodb": "^7.5.0",
|
|
28
|
+
"mongodb-memory-server": "^11.2.0",
|
|
29
29
|
"typescript": "^6.0.3"
|
|
30
30
|
},
|
|
31
31
|
"exports": {
|
|
32
32
|
".": "./dist/src/index.js"
|
|
33
33
|
}
|
|
34
|
-
}
|
|
34
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -9,9 +9,6 @@ export { MongoDBAgentStorageService } from './mongodb-agent-storage-service.js'
|
|
|
9
9
|
export { MongoDBAgentRunService } from './mongodb-agent-run-service.js'
|
|
10
10
|
export { MongoDBSecretService } from './mongodb-secret-service.js'
|
|
11
11
|
|
|
12
|
-
export type { WorkflowRunService } from '@pikku/core/
|
|
13
|
-
export type {
|
|
14
|
-
AgentRunService,
|
|
15
|
-
AgentRunRow,
|
|
16
|
-
} from '@pikku/core/ecosystem/agent'
|
|
12
|
+
export type { WorkflowRunService } from '@pikku/core/workflow'
|
|
13
|
+
export type { AgentRunService, AgentRunRow } from '@pikku/core/agent'
|
|
17
14
|
export { MongoDBSessionStore } from './mongodb-session-store.js'
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import type { AgentThread, AgentMessage } from '@pikku/core/agent'
|
|
2
1
|
import type {
|
|
2
|
+
AgentThread,
|
|
3
|
+
AgentMessage,
|
|
3
4
|
AgentRunRow,
|
|
4
5
|
AgentRunService,
|
|
5
|
-
} from '@pikku/core/
|
|
6
|
+
} from '@pikku/core/agent'
|
|
6
7
|
import type { Db, Collection } from 'mongodb'
|
|
7
8
|
|
|
8
9
|
// Owner ids are untrusted input to the regex, so metacharacters must not be
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import type {
|
|
2
|
-
AgentStorageService,
|
|
3
|
-
AgentRunStateService,
|
|
4
|
-
CreateRunInput,
|
|
5
|
-
SaveScoreInput,
|
|
6
|
-
} from '@pikku/core/services'
|
|
7
|
-
import type {
|
|
8
|
-
AgentThread,
|
|
9
2
|
AgentMessage,
|
|
10
3
|
AgentRunState,
|
|
4
|
+
AgentThread,
|
|
11
5
|
} from '@pikku/core/agent'
|
|
12
6
|
import type { AgentRunScore } from '@pikku/core/agent-scorer'
|
|
7
|
+
import type {
|
|
8
|
+
AgentRunStateService,
|
|
9
|
+
AgentStorageService,
|
|
10
|
+
CreateRunInput,
|
|
11
|
+
SaveScoreInput,
|
|
12
|
+
} from '@pikku/core/services'
|
|
13
13
|
import type { Db, Collection } from 'mongodb'
|
|
14
14
|
|
|
15
15
|
interface AgentThreadDoc {
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
import { buildRemoteHeaders } from '@pikku/core/
|
|
1
|
+
import { buildRemoteHeaders } from '@pikku/core/remote'
|
|
2
2
|
import type {
|
|
3
3
|
DeploymentService,
|
|
4
4
|
DeploymentServiceConfig,
|
|
5
5
|
DeploymentConfig,
|
|
6
|
-
} from '@pikku/core/
|
|
7
|
-
import type { JWTService
|
|
8
|
-
import {
|
|
6
|
+
} from '@pikku/core/services'
|
|
7
|
+
import type { JWTService } from '@pikku/core/services'
|
|
8
|
+
import type { SecretService } from '@pikku/core/services'
|
|
9
|
+
import { getAllFunctionNames } from '@pikku/core/function'
|
|
9
10
|
import type { Db, Collection } from 'mongodb'
|
|
10
11
|
|
|
11
12
|
interface DeploymentDoc {
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { createSecretValue, isSecretValue, type SecretValue } from '@pikku/core'
|
|
2
|
-
import type { SecretService, SecretValues } from '@pikku/core/services'
|
|
3
|
-
import type { Db, Collection } from 'mongodb'
|
|
4
1
|
import {
|
|
5
2
|
deriveKEK,
|
|
6
|
-
generateKEKSalt,
|
|
7
|
-
envelopeEncrypt,
|
|
8
3
|
envelopeDecrypt,
|
|
4
|
+
envelopeEncrypt,
|
|
9
5
|
envelopeRewrap,
|
|
6
|
+
generateKEKSalt,
|
|
10
7
|
} from '@pikku/core/crypto-utils'
|
|
8
|
+
import { createSecretValue, isSecretValue } from '@pikku/core/classification'
|
|
9
|
+
import type { SecretService, SecretValues } from '@pikku/core/services'
|
|
10
|
+
import type { SecretValue } from '@pikku/core/classification'
|
|
11
|
+
import type { Db, Collection } from 'mongodb'
|
|
11
12
|
|
|
12
13
|
export interface MongoDBSecretServiceConfig {
|
|
13
14
|
key: string
|
|
@@ -2,7 +2,7 @@ import { describe, test, before, after } from 'node:test'
|
|
|
2
2
|
import assert from 'node:assert/strict'
|
|
3
3
|
import { MongoClient, type Db } from 'mongodb'
|
|
4
4
|
import { MongoMemoryServer } from 'mongodb-memory-server'
|
|
5
|
-
import { defineServiceTests } from '@pikku/core/
|
|
5
|
+
import { defineServiceTests } from '@pikku/core/testing'
|
|
6
6
|
|
|
7
7
|
import { MongoDBChannelStore } from './mongodb-channel-store.js'
|
|
8
8
|
import { MongoDBEventHubStore } from './mongodb-eventhub-store.js'
|
|
@@ -3,6 +3,7 @@ import assert from 'node:assert/strict'
|
|
|
3
3
|
import { MongoClient, type Db } from 'mongodb'
|
|
4
4
|
import { MongoMemoryServer } from 'mongodb-memory-server'
|
|
5
5
|
|
|
6
|
+
import type { StepState } from '@pikku/core/workflow'
|
|
6
7
|
import { MongoDBWorkflowService } from './mongodb-workflow-service.js'
|
|
7
8
|
|
|
8
9
|
let mongod: MongoMemoryServer
|
|
@@ -119,3 +120,119 @@ describe('a step transition reaches the history', () => {
|
|
|
119
120
|
assert.equal(state.attemptCount, 2)
|
|
120
121
|
})
|
|
121
122
|
})
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Dispatch is at-least-once — the relay re-dispatches a step it believes was
|
|
126
|
+
* dropped, and a queue can redeliver a job it already handed out — so the claim
|
|
127
|
+
* is the only thing standing between a duplicate dispatch and a second
|
|
128
|
+
* execution of a side-effecting step.
|
|
129
|
+
*/
|
|
130
|
+
describe('claiming a step for execution', () => {
|
|
131
|
+
const claim = (runId: string, stepName: string, rpcName = 'rpc.fn') =>
|
|
132
|
+
(ws as any).claimStepForExecution(
|
|
133
|
+
runId,
|
|
134
|
+
stepName,
|
|
135
|
+
rpcName
|
|
136
|
+
) as Promise<StepState | null>
|
|
137
|
+
|
|
138
|
+
const seedRun = () =>
|
|
139
|
+
ws.createRun('flow', {}, false, 'hash', { type: 'test' } as any)
|
|
140
|
+
|
|
141
|
+
test('two dispatches racing for the same pending step: exactly one wins', async () => {
|
|
142
|
+
const runId = await seedRun()
|
|
143
|
+
await ws.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
|
|
144
|
+
|
|
145
|
+
const claims = await Promise.all([claim(runId, 's1'), claim(runId, 's1')])
|
|
146
|
+
const winners = claims.filter((c) => c !== null)
|
|
147
|
+
|
|
148
|
+
assert.equal(
|
|
149
|
+
winners.length,
|
|
150
|
+
1,
|
|
151
|
+
`both dispatches claimed the step, so a side-effecting step would run twice: ${JSON.stringify(claims)}`
|
|
152
|
+
)
|
|
153
|
+
assert.equal((await ws.getStepState(runId, 's1')).status, 'running')
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
test('two dispatches racing to retry the same failed step: exactly one wins', async () => {
|
|
157
|
+
const runId = await seedRun()
|
|
158
|
+
const step = await ws.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
|
|
159
|
+
await ws.setStepRunning(step.stepId)
|
|
160
|
+
await ws.setStepError(step.stepId, new Error('boom'))
|
|
161
|
+
|
|
162
|
+
const claims = await Promise.all([claim(runId, 's1'), claim(runId, 's1')])
|
|
163
|
+
const winners = claims.filter((c) => c !== null)
|
|
164
|
+
|
|
165
|
+
assert.equal(
|
|
166
|
+
winners.length,
|
|
167
|
+
1,
|
|
168
|
+
`both dispatches started a retry, so the failed step would be retried twice concurrently: ${JSON.stringify(claims)}`
|
|
169
|
+
)
|
|
170
|
+
assert.equal(
|
|
171
|
+
(await ws.getStepState(runId, 's1')).attemptCount,
|
|
172
|
+
2,
|
|
173
|
+
'the retry raced itself and burned more than one attempt'
|
|
174
|
+
)
|
|
175
|
+
})
|
|
176
|
+
|
|
177
|
+
test('two dispatches racing for the same scheduled step: exactly one wins', async () => {
|
|
178
|
+
const runId = await seedRun()
|
|
179
|
+
const step = await ws.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
|
|
180
|
+
await ws.setStepScheduled(step.stepId)
|
|
181
|
+
|
|
182
|
+
const claims = await Promise.all([claim(runId, 's1'), claim(runId, 's1')])
|
|
183
|
+
const winners = claims.filter((c) => c !== null)
|
|
184
|
+
|
|
185
|
+
assert.equal(
|
|
186
|
+
winners.length,
|
|
187
|
+
1,
|
|
188
|
+
`both dispatches claimed the queued step, so a redelivered job would run it a second time: ${JSON.stringify(claims)}`
|
|
189
|
+
)
|
|
190
|
+
assert.equal((await ws.getStepState(runId, 's1')).status, 'running')
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
test('a step already claimed is not claimable again', async () => {
|
|
194
|
+
const runId = await seedRun()
|
|
195
|
+
await ws.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
|
|
196
|
+
|
|
197
|
+
assert.notEqual(await claim(runId, 's1'), null)
|
|
198
|
+
assert.equal(
|
|
199
|
+
await claim(runId, 's1'),
|
|
200
|
+
null,
|
|
201
|
+
'a redelivered job re-claimed a step that is already running'
|
|
202
|
+
)
|
|
203
|
+
})
|
|
204
|
+
|
|
205
|
+
test('a succeeded step is not claimable', async () => {
|
|
206
|
+
const runId = await seedRun()
|
|
207
|
+
const step = await ws.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
|
|
208
|
+
await ws.setStepRunning(step.stepId)
|
|
209
|
+
await ws.setStepResult(step.stepId, { ok: true })
|
|
210
|
+
|
|
211
|
+
assert.equal(await claim(runId, 's1'), null)
|
|
212
|
+
})
|
|
213
|
+
|
|
214
|
+
test('a claim that fails releases the step, so it is retryable', async () => {
|
|
215
|
+
const runId = await seedRun()
|
|
216
|
+
const step = await ws.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
|
|
217
|
+
|
|
218
|
+
assert.notEqual(await claim(runId, 's1'), null)
|
|
219
|
+
await ws.setStepError(step.stepId, new Error('boom'))
|
|
220
|
+
|
|
221
|
+
const retry = await claim(runId, 's1')
|
|
222
|
+
assert.notEqual(
|
|
223
|
+
retry,
|
|
224
|
+
null,
|
|
225
|
+
'the failed step stayed claimed, so it can never be retried'
|
|
226
|
+
)
|
|
227
|
+
assert.equal(retry?.status, 'running')
|
|
228
|
+
assert.equal(retry?.attemptCount, 2)
|
|
229
|
+
})
|
|
230
|
+
|
|
231
|
+
test('a claim for a different function than the step was dispatched with is refused', async () => {
|
|
232
|
+
const runId = await seedRun()
|
|
233
|
+
await ws.insertStepState(runId, 's1', 'rpc.fn', { x: 1 })
|
|
234
|
+
|
|
235
|
+
await assert.rejects(() => claim(runId, 's1', 'rpc.other'))
|
|
236
|
+
assert.equal((await ws.getStepState(runId, 's1')).status, 'pending')
|
|
237
|
+
})
|
|
238
|
+
})
|
|
@@ -1,6 +1,18 @@
|
|
|
1
|
-
import type { SerializedError } from '@pikku/core/
|
|
2
|
-
import {
|
|
3
|
-
|
|
1
|
+
import type { SerializedError } from '@pikku/core/errors'
|
|
2
|
+
import {
|
|
3
|
+
PikkuWorkflowService,
|
|
4
|
+
WorkflowStepFunctionMismatchError,
|
|
5
|
+
} from '@pikku/core/workflow'
|
|
6
|
+
import type {
|
|
7
|
+
WorkflowPlannedStep,
|
|
8
|
+
WorkflowQueueOptions,
|
|
9
|
+
WorkflowRun,
|
|
10
|
+
WorkflowRunWire,
|
|
11
|
+
StepState,
|
|
12
|
+
StepStatus,
|
|
13
|
+
WorkflowStatus,
|
|
14
|
+
WorkflowVersionStatus,
|
|
15
|
+
} from '@pikku/core/workflow'
|
|
4
16
|
import type { Db, Collection } from 'mongodb'
|
|
5
17
|
import { MongoDBWorkflowRunService } from './mongodb-workflow-run-service.js'
|
|
6
18
|
|
|
@@ -435,6 +447,12 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
|
|
|
435
447
|
return fn()
|
|
436
448
|
}
|
|
437
449
|
|
|
450
|
+
/**
|
|
451
|
+
* A pass-through: MongoDB has no advisory-lock primitive to build one on.
|
|
452
|
+
* The one decision that must exclude — claiming a step to execute it — is
|
|
453
|
+
* made by `claimStepForExecution` below as a single conditional write, which
|
|
454
|
+
* needs no lock to be exclusive.
|
|
455
|
+
*/
|
|
438
456
|
async withStepLock<T>(
|
|
439
457
|
_runId: string,
|
|
440
458
|
_stepName: string,
|
|
@@ -443,6 +461,74 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
|
|
|
443
461
|
return fn()
|
|
444
462
|
}
|
|
445
463
|
|
|
464
|
+
/**
|
|
465
|
+
* Claim the step with a status-guarded update and read the modified count:
|
|
466
|
+
* a single-document update is atomic in MongoDB, so two dispatches racing for
|
|
467
|
+
* the same step cannot both proceed.
|
|
468
|
+
*
|
|
469
|
+
* This replaces the read-then-write the base engine does under
|
|
470
|
+
* `withStepLock`, which excludes nothing here — the lock above is a
|
|
471
|
+
* pass-through. The winner then goes through the ordinary transition methods,
|
|
472
|
+
* so history rows see exactly what they saw before.
|
|
473
|
+
*/
|
|
474
|
+
protected override async claimStepForExecution(
|
|
475
|
+
runId: string,
|
|
476
|
+
stepName: string,
|
|
477
|
+
rpcName: string
|
|
478
|
+
): Promise<StepState | null> {
|
|
479
|
+
const stepState = await this.getStepState(runId, stepName)
|
|
480
|
+
|
|
481
|
+
// knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
|
|
482
|
+
if (
|
|
483
|
+
stepState.rpcName !== undefined &&
|
|
484
|
+
stepState.rpcName !== (rpcName ?? null)
|
|
485
|
+
) {
|
|
486
|
+
throw new WorkflowStepFunctionMismatchError(runId, stepName)
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
if (stepState.status === 'succeeded' || stepState.status === 'running') {
|
|
490
|
+
return null
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
if (stepState.status === 'failed') {
|
|
494
|
+
if (!(await this.claimStepStatus(stepState.stepId, ['failed']))) {
|
|
495
|
+
return null
|
|
496
|
+
}
|
|
497
|
+
return this.createRetryAttempt(stepState.stepId, 'running')
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
if (stepState.status === 'pending' || stepState.status === 'scheduled') {
|
|
501
|
+
if (
|
|
502
|
+
!(await this.claimStepStatus(stepState.stepId, [
|
|
503
|
+
'pending',
|
|
504
|
+
'scheduled',
|
|
505
|
+
]))
|
|
506
|
+
) {
|
|
507
|
+
return null
|
|
508
|
+
}
|
|
509
|
+
await this.setStepRunning(stepState.stepId)
|
|
510
|
+
return { ...stepState, status: 'running' }
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
return stepState
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Move a step to `running` only if it is still in one of `from`, reporting
|
|
518
|
+
* whether this caller is the one that moved it.
|
|
519
|
+
*/
|
|
520
|
+
private async claimStepStatus(
|
|
521
|
+
stepId: string,
|
|
522
|
+
from: StepStatus[]
|
|
523
|
+
): Promise<boolean> {
|
|
524
|
+
const claimed = await this.steps.updateOne(
|
|
525
|
+
{ _id: stepId, status: { $in: from } },
|
|
526
|
+
{ $set: { status: 'running', updatedAt: new Date() } }
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
return claimed.modifiedCount > 0
|
|
530
|
+
}
|
|
531
|
+
|
|
446
532
|
async getCompletedGraphState(runId: string): Promise<{
|
|
447
533
|
completedNodeIds: string[]
|
|
448
534
|
failedNodeIds: string[]
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
import type { AIStorageService, AIRunStateService, CreateRunInput, SaveScoreInput } from '@pikku/core/services';
|
|
2
|
-
import type { AIThread, AIMessage, AgentRunState } from '@pikku/core/ai-agent';
|
|
3
|
-
import type { AIRunScore } from '@pikku/core/ai-scorer';
|
|
4
|
-
import type { Db } from 'mongodb';
|
|
5
|
-
export declare class MongoDBAIStorageService implements AIStorageService, AIRunStateService {
|
|
6
|
-
private db;
|
|
7
|
-
private initialized;
|
|
8
|
-
private threads;
|
|
9
|
-
private aiMessages;
|
|
10
|
-
private toolCalls;
|
|
11
|
-
private workingMemory;
|
|
12
|
-
private aiRuns;
|
|
13
|
-
private aiRunScores;
|
|
14
|
-
constructor(db: Db);
|
|
15
|
-
init(): Promise<void>;
|
|
16
|
-
createThread(resourceId: string, options?: {
|
|
17
|
-
threadId?: string;
|
|
18
|
-
title?: string;
|
|
19
|
-
metadata?: Record<string, unknown>;
|
|
20
|
-
}): Promise<AIThread>;
|
|
21
|
-
getThread(threadId: string): Promise<AIThread>;
|
|
22
|
-
getThreads(resourceId: string): Promise<AIThread[]>;
|
|
23
|
-
deleteThread(threadId: string): Promise<void>;
|
|
24
|
-
getMessages(threadId: string, options?: {
|
|
25
|
-
lastN?: number;
|
|
26
|
-
cursor?: string;
|
|
27
|
-
}): Promise<AIMessage[]>;
|
|
28
|
-
saveMessages(threadId: string, messages: AIMessage[]): Promise<void>;
|
|
29
|
-
getWorkingMemory(id: string, scope: 'resource' | 'thread'): Promise<Record<string, unknown> | null>;
|
|
30
|
-
saveWorkingMemory(id: string, scope: 'resource' | 'thread', data: Record<string, unknown>): Promise<void>;
|
|
31
|
-
createRun(run: CreateRunInput): Promise<string>;
|
|
32
|
-
updateRun(runId: string, updates: Partial<AgentRunState>): Promise<void>;
|
|
33
|
-
getRun(runId: string): Promise<AgentRunState | null>;
|
|
34
|
-
getRunsByThread(threadId: string): Promise<AgentRunState[]>;
|
|
35
|
-
private setApprovals;
|
|
36
|
-
private mapRunRow;
|
|
37
|
-
findRunByToolCallId(toolCallId: string): Promise<{
|
|
38
|
-
run: AgentRunState;
|
|
39
|
-
approval: NonNullable<AgentRunState['pendingApprovals']>[number];
|
|
40
|
-
} | null>;
|
|
41
|
-
resolveApproval(toolCallId: string, status: 'approved' | 'denied'): Promise<boolean>;
|
|
42
|
-
saveScore(score: SaveScoreInput): Promise<void>;
|
|
43
|
-
getScores(runId: string): Promise<AIRunScore[]>;
|
|
44
|
-
close(): Promise<void>;
|
|
45
|
-
}
|
|
@@ -1,438 +0,0 @@
|
|
|
1
|
-
export class MongoDBAIStorageService {
|
|
2
|
-
db;
|
|
3
|
-
initialized = false;
|
|
4
|
-
threads;
|
|
5
|
-
aiMessages;
|
|
6
|
-
toolCalls;
|
|
7
|
-
workingMemory;
|
|
8
|
-
aiRuns;
|
|
9
|
-
aiRunScores;
|
|
10
|
-
constructor(db) {
|
|
11
|
-
this.db = db;
|
|
12
|
-
}
|
|
13
|
-
async init() {
|
|
14
|
-
if (this.initialized)
|
|
15
|
-
return;
|
|
16
|
-
this.threads = this.db.collection('ai_threads');
|
|
17
|
-
this.aiMessages = this.db.collection('ai_message');
|
|
18
|
-
this.toolCalls = this.db.collection('ai_tool_call');
|
|
19
|
-
this.workingMemory =
|
|
20
|
-
this.db.collection('ai_working_memory');
|
|
21
|
-
this.aiRuns = this.db.collection('ai_run');
|
|
22
|
-
this.aiRunScores = this.db.collection('ai_run_score');
|
|
23
|
-
await this.threads.createIndex({ resourceId: 1 });
|
|
24
|
-
await this.aiMessages.createIndex({ threadId: 1, createdAt: 1 });
|
|
25
|
-
await this.toolCalls.createIndex({ threadId: 1 });
|
|
26
|
-
await this.toolCalls.createIndex({ messageId: 1 });
|
|
27
|
-
await this.workingMemory.createIndex({ id: 1, scope: 1 }, { unique: true });
|
|
28
|
-
await this.aiRuns.createIndex({ threadId: 1, createdAt: 1 });
|
|
29
|
-
await this.aiRunScores.createIndex({ runId: 1, createdAt: 1 });
|
|
30
|
-
this.initialized = true;
|
|
31
|
-
}
|
|
32
|
-
async createThread(resourceId, options) {
|
|
33
|
-
const id = options?.threadId ?? crypto.randomUUID();
|
|
34
|
-
const now = new Date();
|
|
35
|
-
await this.threads.insertOne({
|
|
36
|
-
_id: id,
|
|
37
|
-
resourceId,
|
|
38
|
-
title: options?.title ?? null,
|
|
39
|
-
metadata: options?.metadata ?? null,
|
|
40
|
-
createdAt: now,
|
|
41
|
-
updatedAt: now,
|
|
42
|
-
});
|
|
43
|
-
return {
|
|
44
|
-
id,
|
|
45
|
-
resourceId,
|
|
46
|
-
title: options?.title,
|
|
47
|
-
metadata: options?.metadata,
|
|
48
|
-
createdAt: now,
|
|
49
|
-
updatedAt: now,
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
|
-
async getThread(threadId) {
|
|
53
|
-
const row = await this.threads.findOne({ _id: threadId });
|
|
54
|
-
if (!row) {
|
|
55
|
-
throw new Error(`Thread not found: ${threadId}`);
|
|
56
|
-
}
|
|
57
|
-
return {
|
|
58
|
-
id: row._id,
|
|
59
|
-
resourceId: row.resourceId,
|
|
60
|
-
title: row.title ?? undefined,
|
|
61
|
-
metadata: row.metadata ?? undefined,
|
|
62
|
-
createdAt: new Date(row.createdAt),
|
|
63
|
-
updatedAt: new Date(row.updatedAt),
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
async getThreads(resourceId) {
|
|
67
|
-
const result = await this.threads
|
|
68
|
-
.find({ resourceId })
|
|
69
|
-
.sort({ updatedAt: -1 })
|
|
70
|
-
.toArray();
|
|
71
|
-
return result.map((row) => ({
|
|
72
|
-
id: row._id,
|
|
73
|
-
resourceId: row.resourceId,
|
|
74
|
-
title: row.title ?? undefined,
|
|
75
|
-
metadata: row.metadata ?? undefined,
|
|
76
|
-
createdAt: new Date(row.createdAt),
|
|
77
|
-
updatedAt: new Date(row.updatedAt),
|
|
78
|
-
}));
|
|
79
|
-
}
|
|
80
|
-
async deleteThread(threadId) {
|
|
81
|
-
await this.toolCalls.deleteMany({ threadId });
|
|
82
|
-
await this.aiMessages.deleteMany({ threadId });
|
|
83
|
-
await this.aiRuns.deleteMany({ threadId });
|
|
84
|
-
await this.threads.deleteOne({ _id: threadId });
|
|
85
|
-
}
|
|
86
|
-
async getMessages(threadId, options) {
|
|
87
|
-
let filter = { threadId };
|
|
88
|
-
if (options?.cursor) {
|
|
89
|
-
const cursorRow = await this.aiMessages.findOne({ _id: options.cursor });
|
|
90
|
-
if (cursorRow) {
|
|
91
|
-
filter.createdAt = { $lt: cursorRow.createdAt };
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
let msgResult;
|
|
95
|
-
if (options?.cursor || options?.lastN) {
|
|
96
|
-
const innerResult = await this.aiMessages
|
|
97
|
-
.find(filter)
|
|
98
|
-
.sort({ createdAt: -1 })
|
|
99
|
-
.limit(options?.lastN ?? 50)
|
|
100
|
-
.toArray();
|
|
101
|
-
innerResult.reverse();
|
|
102
|
-
msgResult = innerResult;
|
|
103
|
-
}
|
|
104
|
-
else {
|
|
105
|
-
msgResult = await this.aiMessages
|
|
106
|
-
.find(filter)
|
|
107
|
-
.sort({ createdAt: 1 })
|
|
108
|
-
.toArray();
|
|
109
|
-
}
|
|
110
|
-
const tcResult = await this.toolCalls
|
|
111
|
-
.find({ threadId })
|
|
112
|
-
.sort({ createdAt: 1 })
|
|
113
|
-
.toArray();
|
|
114
|
-
const tcByMessage = new Map();
|
|
115
|
-
for (const tc of tcResult) {
|
|
116
|
-
const msgId = tc.messageId;
|
|
117
|
-
if (!tcByMessage.has(msgId))
|
|
118
|
-
tcByMessage.set(msgId, []);
|
|
119
|
-
tcByMessage.get(msgId).push(tc);
|
|
120
|
-
}
|
|
121
|
-
const messages = [];
|
|
122
|
-
for (const row of msgResult) {
|
|
123
|
-
let parsedContent = row.content ?? undefined;
|
|
124
|
-
if (typeof row.content === 'string') {
|
|
125
|
-
try {
|
|
126
|
-
const parsed = JSON.parse(row.content);
|
|
127
|
-
if (Array.isArray(parsed)) {
|
|
128
|
-
parsedContent = parsed;
|
|
129
|
-
}
|
|
130
|
-
else if (typeof parsed === 'string') {
|
|
131
|
-
parsedContent = parsed;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
catch {
|
|
135
|
-
// Not JSON, use raw string
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
const msg = {
|
|
139
|
-
id: row._id,
|
|
140
|
-
role: row.role,
|
|
141
|
-
content: parsedContent,
|
|
142
|
-
createdAt: new Date(row.createdAt),
|
|
143
|
-
};
|
|
144
|
-
const tcs = tcByMessage.get(msg.id);
|
|
145
|
-
if (tcs?.length) {
|
|
146
|
-
msg.toolCalls = tcs.map((tc) => ({
|
|
147
|
-
id: tc._id,
|
|
148
|
-
name: tc.toolName,
|
|
149
|
-
args: tc.args,
|
|
150
|
-
}));
|
|
151
|
-
const completed = tcs.filter((tc) => tc.result != null);
|
|
152
|
-
if (completed.length) {
|
|
153
|
-
messages.push(msg);
|
|
154
|
-
messages.push({
|
|
155
|
-
id: `tool-results-${msg.id}`,
|
|
156
|
-
role: 'tool',
|
|
157
|
-
toolResults: completed.map((tc) => ({
|
|
158
|
-
id: tc._id,
|
|
159
|
-
name: tc.toolName,
|
|
160
|
-
result: tc.result,
|
|
161
|
-
})),
|
|
162
|
-
createdAt: msg.createdAt,
|
|
163
|
-
});
|
|
164
|
-
continue;
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
messages.push(msg);
|
|
168
|
-
}
|
|
169
|
-
return messages;
|
|
170
|
-
}
|
|
171
|
-
async saveMessages(threadId, messages) {
|
|
172
|
-
if (messages.length === 0)
|
|
173
|
-
return;
|
|
174
|
-
const toolMessages = messages.filter((m) => m.role === 'tool');
|
|
175
|
-
const nonToolMessages = messages.filter((m) => m.role !== 'tool');
|
|
176
|
-
if (nonToolMessages.length > 0) {
|
|
177
|
-
await this.aiMessages.insertMany(nonToolMessages.map((msg) => ({
|
|
178
|
-
_id: msg.id,
|
|
179
|
-
threadId,
|
|
180
|
-
role: msg.role,
|
|
181
|
-
content: msg.content != null ? JSON.stringify(msg.content) : null,
|
|
182
|
-
createdAt: msg.createdAt ?? new Date(),
|
|
183
|
-
})));
|
|
184
|
-
}
|
|
185
|
-
const toolCallDocs = nonToolMessages.flatMap((msg) => msg.toolCalls?.map((tc) => ({ ...tc, messageId: msg.id })) ?? []);
|
|
186
|
-
if (toolCallDocs.length > 0) {
|
|
187
|
-
await this.toolCalls.insertMany(toolCallDocs.map((tc) => ({
|
|
188
|
-
_id: tc.id,
|
|
189
|
-
threadId,
|
|
190
|
-
messageId: tc.messageId,
|
|
191
|
-
runId: null,
|
|
192
|
-
toolName: tc.name,
|
|
193
|
-
args: tc.args,
|
|
194
|
-
result: null,
|
|
195
|
-
approvalStatus: null,
|
|
196
|
-
approvalType: null,
|
|
197
|
-
agentRunId: null,
|
|
198
|
-
displayToolName: null,
|
|
199
|
-
displayArgs: null,
|
|
200
|
-
createdAt: new Date(),
|
|
201
|
-
})));
|
|
202
|
-
}
|
|
203
|
-
for (const toolMsg of toolMessages) {
|
|
204
|
-
if (!toolMsg.toolResults)
|
|
205
|
-
continue;
|
|
206
|
-
for (const tr of toolMsg.toolResults) {
|
|
207
|
-
await this.toolCalls.updateOne({ _id: tr.id }, { $set: { result: tr.result } });
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
await this.threads.updateOne({ _id: threadId }, { $set: { updatedAt: new Date() } });
|
|
211
|
-
}
|
|
212
|
-
async getWorkingMemory(id, scope) {
|
|
213
|
-
const row = await this.workingMemory.findOne({ id, scope });
|
|
214
|
-
if (!row)
|
|
215
|
-
return null;
|
|
216
|
-
return row.data;
|
|
217
|
-
}
|
|
218
|
-
async saveWorkingMemory(id, scope, data) {
|
|
219
|
-
await this.workingMemory.updateOne({ id, scope }, {
|
|
220
|
-
$set: {
|
|
221
|
-
data,
|
|
222
|
-
updatedAt: new Date(),
|
|
223
|
-
},
|
|
224
|
-
$setOnInsert: {
|
|
225
|
-
id,
|
|
226
|
-
scope,
|
|
227
|
-
},
|
|
228
|
-
}, { upsert: true });
|
|
229
|
-
}
|
|
230
|
-
async createRun(run) {
|
|
231
|
-
const runId = crypto.randomUUID();
|
|
232
|
-
await this.aiRuns.insertOne({
|
|
233
|
-
_id: runId,
|
|
234
|
-
agentName: run.agentName,
|
|
235
|
-
threadId: run.threadId,
|
|
236
|
-
resourceId: run.resourceId,
|
|
237
|
-
status: run.status,
|
|
238
|
-
errorMessage: run.errorMessage ?? null,
|
|
239
|
-
suspendReason: run.suspendReason ?? null,
|
|
240
|
-
missingRpcs: run.missingRpcs ?? null,
|
|
241
|
-
usageInputTokens: run.usage.inputTokens,
|
|
242
|
-
usageOutputTokens: run.usage.outputTokens,
|
|
243
|
-
usageModel: run.usage.model,
|
|
244
|
-
createdAt: run.createdAt,
|
|
245
|
-
updatedAt: run.updatedAt,
|
|
246
|
-
});
|
|
247
|
-
if (run.pendingApprovals?.length) {
|
|
248
|
-
await this.setApprovals(runId, run.pendingApprovals);
|
|
249
|
-
}
|
|
250
|
-
return runId;
|
|
251
|
-
}
|
|
252
|
-
async updateRun(runId, updates) {
|
|
253
|
-
const setValues = { updatedAt: new Date() };
|
|
254
|
-
if (updates.status !== undefined) {
|
|
255
|
-
setValues.status = updates.status;
|
|
256
|
-
}
|
|
257
|
-
if (updates.errorMessage !== undefined) {
|
|
258
|
-
setValues.errorMessage = updates.errorMessage;
|
|
259
|
-
}
|
|
260
|
-
if (updates.suspendReason !== undefined) {
|
|
261
|
-
setValues.suspendReason = updates.suspendReason;
|
|
262
|
-
}
|
|
263
|
-
if (updates.missingRpcs !== undefined) {
|
|
264
|
-
setValues.missingRpcs = updates.missingRpcs;
|
|
265
|
-
}
|
|
266
|
-
if (updates.usage !== undefined) {
|
|
267
|
-
setValues.usageInputTokens = updates.usage.inputTokens;
|
|
268
|
-
setValues.usageOutputTokens = updates.usage.outputTokens;
|
|
269
|
-
setValues.usageModel = updates.usage.model;
|
|
270
|
-
}
|
|
271
|
-
await this.aiRuns.updateOne({ _id: runId }, { $set: setValues });
|
|
272
|
-
if (updates.pendingApprovals !== undefined) {
|
|
273
|
-
await this.toolCalls.updateMany({ runId, approvalStatus: { $ne: null } }, {
|
|
274
|
-
$set: {
|
|
275
|
-
approvalStatus: null,
|
|
276
|
-
runId: null,
|
|
277
|
-
approvalType: null,
|
|
278
|
-
agentRunId: null,
|
|
279
|
-
displayToolName: null,
|
|
280
|
-
displayArgs: null,
|
|
281
|
-
},
|
|
282
|
-
});
|
|
283
|
-
if (updates.pendingApprovals.length) {
|
|
284
|
-
await this.setApprovals(runId, updates.pendingApprovals);
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
async getRun(runId) {
|
|
289
|
-
const row = await this.aiRuns.findOne({ _id: runId });
|
|
290
|
-
if (!row)
|
|
291
|
-
return null;
|
|
292
|
-
const approvals = await this.toolCalls
|
|
293
|
-
.find({ runId, approvalStatus: 'pending' })
|
|
294
|
-
.toArray();
|
|
295
|
-
return this.mapRunRow(row, approvals);
|
|
296
|
-
}
|
|
297
|
-
async getRunsByThread(threadId) {
|
|
298
|
-
const result = await this.aiRuns
|
|
299
|
-
.find({ threadId })
|
|
300
|
-
.sort({ createdAt: -1 })
|
|
301
|
-
.toArray();
|
|
302
|
-
const runs = [];
|
|
303
|
-
for (const row of result) {
|
|
304
|
-
const approvals = await this.toolCalls
|
|
305
|
-
.find({ runId: row._id, approvalStatus: 'pending' })
|
|
306
|
-
.toArray();
|
|
307
|
-
runs.push(this.mapRunRow(row, approvals));
|
|
308
|
-
}
|
|
309
|
-
return runs;
|
|
310
|
-
}
|
|
311
|
-
async setApprovals(runId, approvals) {
|
|
312
|
-
for (const a of approvals) {
|
|
313
|
-
if (a.type === 'agent-call') {
|
|
314
|
-
await this.toolCalls.updateOne({ _id: a.toolCallId }, {
|
|
315
|
-
$set: {
|
|
316
|
-
approvalStatus: 'pending',
|
|
317
|
-
runId,
|
|
318
|
-
approvalType: 'agent-call',
|
|
319
|
-
agentRunId: a.agentRunId,
|
|
320
|
-
displayToolName: a.displayToolName,
|
|
321
|
-
displayArgs: a.displayArgs,
|
|
322
|
-
},
|
|
323
|
-
});
|
|
324
|
-
}
|
|
325
|
-
else {
|
|
326
|
-
await this.toolCalls.updateOne({ _id: a.toolCallId }, {
|
|
327
|
-
$set: {
|
|
328
|
-
approvalStatus: 'pending',
|
|
329
|
-
runId,
|
|
330
|
-
approvalType: 'tool-call',
|
|
331
|
-
},
|
|
332
|
-
});
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
mapRunRow(row, approvalRows) {
|
|
337
|
-
const pendingApprovals = approvalRows?.length
|
|
338
|
-
? approvalRows.map((a) => {
|
|
339
|
-
if (a.approvalType === 'agent-call') {
|
|
340
|
-
return {
|
|
341
|
-
type: 'agent-call',
|
|
342
|
-
toolCallId: a._id,
|
|
343
|
-
agentName: a.toolName,
|
|
344
|
-
agentRunId: a.agentRunId,
|
|
345
|
-
displayToolName: a.displayToolName,
|
|
346
|
-
displayArgs: a.displayArgs,
|
|
347
|
-
};
|
|
348
|
-
}
|
|
349
|
-
return {
|
|
350
|
-
type: 'tool-call',
|
|
351
|
-
toolCallId: a._id,
|
|
352
|
-
toolName: a.toolName,
|
|
353
|
-
args: a.args,
|
|
354
|
-
};
|
|
355
|
-
})
|
|
356
|
-
: undefined;
|
|
357
|
-
return {
|
|
358
|
-
runId: row._id,
|
|
359
|
-
agentName: row.agentName,
|
|
360
|
-
threadId: row.threadId,
|
|
361
|
-
resourceId: row.resourceId,
|
|
362
|
-
status: row.status,
|
|
363
|
-
errorMessage: row.errorMessage ?? undefined,
|
|
364
|
-
suspendReason: row.suspendReason,
|
|
365
|
-
missingRpcs: row.missingRpcs ?? undefined,
|
|
366
|
-
pendingApprovals,
|
|
367
|
-
usage: {
|
|
368
|
-
inputTokens: Number(row.usageInputTokens),
|
|
369
|
-
outputTokens: Number(row.usageOutputTokens),
|
|
370
|
-
model: row.usageModel,
|
|
371
|
-
},
|
|
372
|
-
createdAt: new Date(row.createdAt),
|
|
373
|
-
updatedAt: new Date(row.updatedAt),
|
|
374
|
-
};
|
|
375
|
-
}
|
|
376
|
-
async findRunByToolCallId(toolCallId) {
|
|
377
|
-
const tc = await this.toolCalls.findOne({
|
|
378
|
-
_id: toolCallId,
|
|
379
|
-
approvalStatus: 'pending',
|
|
380
|
-
});
|
|
381
|
-
if (!tc || !tc.runId)
|
|
382
|
-
return null;
|
|
383
|
-
const run = await this.getRun(tc.runId);
|
|
384
|
-
if (!run)
|
|
385
|
-
return null;
|
|
386
|
-
let approval;
|
|
387
|
-
if (tc.approvalType === 'agent-call') {
|
|
388
|
-
approval = {
|
|
389
|
-
type: 'agent-call',
|
|
390
|
-
toolCallId: tc._id,
|
|
391
|
-
agentName: tc.toolName,
|
|
392
|
-
agentRunId: tc.agentRunId,
|
|
393
|
-
displayToolName: tc.displayToolName,
|
|
394
|
-
displayArgs: tc.displayArgs,
|
|
395
|
-
};
|
|
396
|
-
}
|
|
397
|
-
else {
|
|
398
|
-
approval = {
|
|
399
|
-
type: 'tool-call',
|
|
400
|
-
toolCallId: tc._id,
|
|
401
|
-
toolName: tc.toolName,
|
|
402
|
-
args: tc.args,
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
return { run, approval };
|
|
406
|
-
}
|
|
407
|
-
async resolveApproval(toolCallId, status) {
|
|
408
|
-
// Only the caller that moves the tool call off 'pending' has claimed it.
|
|
409
|
-
const result = await this.toolCalls.updateOne({ _id: toolCallId, approvalStatus: 'pending' }, { $set: { approvalStatus: status } });
|
|
410
|
-
return result.modifiedCount > 0;
|
|
411
|
-
}
|
|
412
|
-
async saveScore(score) {
|
|
413
|
-
await this.aiRunScores.insertOne({
|
|
414
|
-
_id: crypto.randomUUID(),
|
|
415
|
-
runId: score.runId,
|
|
416
|
-
scorerName: score.scorerName,
|
|
417
|
-
score: score.score,
|
|
418
|
-
reason: score.reason ?? null,
|
|
419
|
-
metadata: score.metadata ?? null,
|
|
420
|
-
createdAt: new Date(),
|
|
421
|
-
});
|
|
422
|
-
}
|
|
423
|
-
async getScores(runId) {
|
|
424
|
-
const rows = await this.aiRunScores
|
|
425
|
-
.find({ runId })
|
|
426
|
-
.sort({ createdAt: 1 })
|
|
427
|
-
.toArray();
|
|
428
|
-
return rows.map((row) => ({
|
|
429
|
-
runId: row.runId,
|
|
430
|
-
scorerName: row.scorerName,
|
|
431
|
-
score: row.score,
|
|
432
|
-
reason: row.reason ?? undefined,
|
|
433
|
-
metadata: row.metadata ?? undefined,
|
|
434
|
-
createdAt: row.createdAt,
|
|
435
|
-
}));
|
|
436
|
-
}
|
|
437
|
-
async close() { }
|
|
438
|
-
}
|