@plico-ai/runtime 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +4 -0
- package/README.md +11 -2
- package/dist/artifacts.d.ts +10 -0
- package/dist/artifacts.js +106 -0
- package/dist/engine/context.d.ts +16 -0
- package/dist/engine/context.js +77 -0
- package/dist/engine/engine.d.ts +12 -0
- package/dist/engine/engine.js +235 -0
- package/dist/engine/errors.d.ts +12 -0
- package/dist/engine/errors.js +19 -0
- package/dist/engine/index.d.ts +7 -0
- package/dist/engine/index.js +5 -0
- package/dist/engine/journal.d.ts +65 -0
- package/dist/engine/journal.js +1 -0
- package/dist/engine/memory-journal.d.ts +2 -0
- package/dist/engine/memory-journal.js +125 -0
- package/dist/engine/sqlite-engine.d.ts +7 -0
- package/dist/engine/sqlite-engine.js +6 -0
- package/dist/engine/sqlite-journal.d.ts +5 -0
- package/dist/engine/sqlite-journal.js +136 -0
- package/dist/engine/types.d.ts +97 -0
- package/dist/engine/types.js +1 -0
- package/dist/env.d.ts +7 -0
- package/dist/env.js +19 -0
- package/dist/errors.d.ts +18 -0
- package/dist/errors.js +40 -0
- package/dist/evals/discover.d.ts +3 -0
- package/dist/evals/discover.js +49 -0
- package/dist/evals/index.d.ts +3 -0
- package/dist/evals/index.js +3 -0
- package/dist/evals/promote.d.ts +7 -0
- package/dist/evals/promote.js +32 -0
- package/dist/evals/run-evals.d.ts +2 -0
- package/dist/evals/run-evals.js +69 -0
- package/dist/events.d.ts +28 -0
- package/dist/events.js +104 -0
- package/dist/guards.d.ts +1 -0
- package/dist/guards.js +3 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.js +17 -0
- package/dist/instructions.d.ts +4 -0
- package/dist/instructions.js +23 -0
- package/dist/policy/approval.d.ts +8 -0
- package/dist/policy/approval.js +27 -0
- package/dist/providers/ai-sdk.d.ts +10 -0
- package/dist/providers/ai-sdk.js +154 -0
- package/dist/providers/configured.d.ts +17 -0
- package/dist/providers/configured.js +60 -0
- package/dist/providers/openai-compatible.d.ts +8 -0
- package/dist/providers/openai-compatible.js +14 -0
- package/dist/providers/project-module.d.ts +7 -0
- package/dist/providers/project-module.js +29 -0
- package/dist/providers/registry.d.ts +20 -0
- package/dist/providers/registry.js +53 -0
- package/dist/readiness/check-readiness.d.ts +2 -0
- package/dist/readiness/check-readiness.js +33 -0
- package/dist/readiness/checks/approval-policy.d.ts +5 -0
- package/dist/readiness/checks/approval-policy.js +19 -0
- package/dist/readiness/checks/artifact-access.d.ts +3 -0
- package/dist/readiness/checks/artifact-access.js +5 -0
- package/dist/readiness/checks/artifacts.d.ts +6 -0
- package/dist/readiness/checks/artifacts.js +22 -0
- package/dist/readiness/checks/env.d.ts +3 -0
- package/dist/readiness/checks/env.js +10 -0
- package/dist/readiness/checks/evals.d.ts +2 -0
- package/dist/readiness/checks/evals.js +53 -0
- package/dist/readiness/checks/persistence.d.ts +3 -0
- package/dist/readiness/checks/persistence.js +61 -0
- package/dist/readiness/checks/project.d.ts +7 -0
- package/dist/readiness/checks/project.js +21 -0
- package/dist/readiness/checks/provider.d.ts +2 -0
- package/dist/readiness/checks/provider.js +20 -0
- package/dist/readiness/checks/server-config.d.ts +2 -0
- package/dist/readiness/checks/server-config.js +13 -0
- package/dist/readiness/checks/studio.d.ts +2 -0
- package/dist/readiness/checks/studio.js +16 -0
- package/dist/readiness/checks/thread-store.d.ts +3 -0
- package/dist/readiness/checks/thread-store.js +12 -0
- package/dist/readiness/checks/tools.d.ts +3 -0
- package/dist/readiness/checks/tools.js +42 -0
- package/dist/readiness/diagnostic.d.ts +2 -0
- package/dist/readiness/diagnostic.js +7 -0
- package/dist/readiness/index.d.ts +1 -0
- package/dist/readiness/index.js +1 -0
- package/dist/readiness/open-database.d.ts +8 -0
- package/dist/readiness/open-database.js +15 -0
- package/dist/run/artifact-tool-context.d.ts +19 -0
- package/dist/run/artifact-tool-context.js +74 -0
- package/dist/run/finalize.d.ts +16 -0
- package/dist/run/finalize.js +11 -0
- package/dist/run/index.d.ts +1 -0
- package/dist/run/index.js +1 -0
- package/dist/run/pinned-instructions.d.ts +18 -0
- package/dist/run/pinned-instructions.js +36 -0
- package/dist/run/record-tools.d.ts +8 -0
- package/dist/run/record-tools.js +21 -0
- package/dist/run/run-project.d.ts +17 -0
- package/dist/run/run-project.js +222 -0
- package/dist/run/strategies/approval.d.ts +9 -0
- package/dist/run/strategies/approval.js +42 -0
- package/dist/run/strategies/default-dry.d.ts +3 -0
- package/dist/run/strategies/default-dry.js +5 -0
- package/dist/run/strategies/provider.d.ts +6 -0
- package/dist/run/strategies/provider.js +193 -0
- package/dist/run/strategies/scripted.d.ts +5 -0
- package/dist/run/strategies/scripted.js +106 -0
- package/dist/run/tool-fingerprint.d.ts +2 -0
- package/dist/run/tool-fingerprint.js +26 -0
- package/dist/run/tool-step.d.ts +4 -0
- package/dist/run/tool-step.js +62 -0
- package/dist/run/tracing.d.ts +12 -0
- package/dist/run/tracing.js +92 -0
- package/dist/script.d.ts +4 -0
- package/dist/script.js +51 -0
- package/dist/storage/index.d.ts +1 -0
- package/dist/storage/index.js +1 -0
- package/dist/storage/metadata.d.ts +5 -0
- package/dist/storage/metadata.js +26 -0
- package/dist/storage/sqlite/append-event.d.ts +3 -0
- package/dist/storage/sqlite/append-event.js +108 -0
- package/dist/storage/sqlite/driver.d.ts +9 -0
- package/dist/storage/sqlite/driver.js +13 -0
- package/dist/storage/sqlite/index.d.ts +1 -0
- package/dist/storage/sqlite/index.js +1 -0
- package/dist/storage/sqlite/open.d.ts +3 -0
- package/dist/storage/sqlite/open.js +54 -0
- package/dist/storage/sqlite/owner-lock.d.ts +5 -0
- package/dist/storage/sqlite/owner-lock.js +55 -0
- package/dist/storage/sqlite/project-thread.d.ts +4 -0
- package/dist/storage/sqlite/project-thread.js +33 -0
- package/dist/storage/sqlite/queries.d.ts +9 -0
- package/dist/storage/sqlite/queries.js +134 -0
- package/dist/storage/sqlite/recover-interrupted.d.ts +2 -0
- package/dist/storage/sqlite/recover-interrupted.js +46 -0
- package/dist/storage/sqlite/schema.d.ts +10 -0
- package/dist/storage/sqlite/schema.js +205 -0
- package/dist/storage/sqlite/store.d.ts +2 -0
- package/dist/storage/sqlite/store.js +51 -0
- package/dist/storage/sqlite/thread-log.d.ts +4 -0
- package/dist/storage/sqlite/thread-log.js +9 -0
- package/dist/storage/sqlite/thread-title.d.ts +24 -0
- package/dist/storage/sqlite/thread-title.js +85 -0
- package/dist/storage/sqlite/threads.d.ts +47 -0
- package/dist/storage/sqlite/threads.js +156 -0
- package/dist/threads/stopwords.d.ts +1 -0
- package/dist/threads/stopwords.js +16 -0
- package/dist/threads/suggest-title.d.ts +9 -0
- package/dist/threads/suggest-title.js +102 -0
- package/dist/threads/titler.d.ts +10 -0
- package/dist/threads/titler.js +63 -0
- package/dist/threads/words.d.ts +12 -0
- package/dist/threads/words.js +39 -0
- package/dist/tools/artifact-model-tool-operations.d.ts +29 -0
- package/dist/tools/artifact-model-tool-operations.js +124 -0
- package/dist/tools/artifact-model-tools.d.ts +2 -0
- package/dist/tools/artifact-model-tools.js +101 -0
- package/dist/tools/discover.d.ts +4 -0
- package/dist/tools/discover.js +34 -0
- package/dist/tools/index.d.ts +3 -0
- package/dist/tools/index.js +3 -0
- package/dist/tools/load.d.ts +2 -0
- package/dist/tools/load.js +83 -0
- package/dist/tools/normalize.d.ts +2 -0
- package/dist/tools/normalize.js +48 -0
- package/dist/tools/schema.d.ts +7 -0
- package/dist/tools/schema.js +101 -0
- package/dist/tools/skill-model-tool.d.ts +7 -0
- package/dist/tools/skill-model-tool.js +38 -0
- package/dist/types/artifacts.d.ts +5 -0
- package/dist/types/artifacts.js +1 -0
- package/dist/types/evals.d.ts +16 -0
- package/dist/types/evals.js +1 -0
- package/dist/types/index.d.ts +10 -0
- package/dist/types/index.js +1 -0
- package/dist/types/provider.d.ts +45 -0
- package/dist/types/provider.js +1 -0
- package/dist/types/readiness.d.ts +9 -0
- package/dist/types/readiness.js +1 -0
- package/dist/types/run.d.ts +28 -0
- package/dist/types/run.js +1 -0
- package/dist/types/script.d.ts +16 -0
- package/dist/types/script.js +1 -0
- package/dist/types/store.d.ts +24 -0
- package/dist/types/store.js +1 -0
- package/dist/types/tools.d.ts +15 -0
- package/dist/types/tools.js +1 -0
- package/package.json +77 -3
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
export interface StepOptions {
|
|
2
|
+
/** Extra attempts inside one step. Defaults to 0: the step runs once unless the process dies inside it. */
|
|
3
|
+
retries?: number;
|
|
4
|
+
/** When supplied, only errors accepted by this predicate are retried. */
|
|
5
|
+
shouldRetry?: (error: unknown) => boolean;
|
|
6
|
+
}
|
|
7
|
+
export interface StepAttempt {
|
|
8
|
+
/** Zero-based attempt number within this step. */
|
|
9
|
+
attempt: number;
|
|
10
|
+
/** Aborts when the run is cancelled, so a long step can stop early. */
|
|
11
|
+
signal: AbortSignal;
|
|
12
|
+
}
|
|
13
|
+
export interface RunContext {
|
|
14
|
+
readonly runId: string;
|
|
15
|
+
/** Aborts when the run is cancelled. */
|
|
16
|
+
readonly signal: AbortSignal;
|
|
17
|
+
/**
|
|
18
|
+
* Runs `fn` and records its result. When the run is replayed, the recorded result is returned
|
|
19
|
+
* and `fn` does not run. Results are stored as JSON, so `undefined` properties are dropped.
|
|
20
|
+
* A step runs at least once: a process that dies inside it runs it again on recovery.
|
|
21
|
+
*/
|
|
22
|
+
step<T>(name: string, fn: (attempt: StepAttempt) => T | Promise<T>, options?: StepOptions): Promise<T>;
|
|
23
|
+
/**
|
|
24
|
+
* Holds the run until a signal named `name` arrives or the timeout passes. A signal that arrived
|
|
25
|
+
* earlier is used at once. Nothing executes while the run waits: the first time there is nothing
|
|
26
|
+
* to return, this throws RunSuspended. Let it reach the engine, or catch it and return; the engine
|
|
27
|
+
* replays the workflow from its recorded steps once the wait can end. Use a unique name per wait.
|
|
28
|
+
*/
|
|
29
|
+
wait<T = unknown>(name: string, options?: WaitOptions): Promise<WaitOutcome<T>>;
|
|
30
|
+
}
|
|
31
|
+
/** How a run ended for now. A waiting run may carry what the workflow returned before it stopped. */
|
|
32
|
+
export type RunOutcome<O> = {
|
|
33
|
+
state: "completed";
|
|
34
|
+
output: O;
|
|
35
|
+
} | {
|
|
36
|
+
state: "waiting";
|
|
37
|
+
output?: O;
|
|
38
|
+
} | {
|
|
39
|
+
state: "cancelled";
|
|
40
|
+
output?: O;
|
|
41
|
+
};
|
|
42
|
+
/** `ignored`: the run had already finished or been cancelled. */
|
|
43
|
+
export type CancelResult = "accepted" | "ignored" | "unknown";
|
|
44
|
+
/** `ignored`: a decision arrived late, twice, or for a finished run. The first one wins. */
|
|
45
|
+
export type SignalResult = "accepted" | "ignored" | "unknown";
|
|
46
|
+
export interface WaitOptions {
|
|
47
|
+
/** Ends the wait as a timeout this long after it is first reached. No timeout by default. */
|
|
48
|
+
timeoutMs?: number;
|
|
49
|
+
}
|
|
50
|
+
/** A timeout is its own outcome, so a caller can tell it from a decision. */
|
|
51
|
+
export type WaitOutcome<T> = {
|
|
52
|
+
kind: "signal";
|
|
53
|
+
value: T;
|
|
54
|
+
} | {
|
|
55
|
+
kind: "timeout";
|
|
56
|
+
};
|
|
57
|
+
export type Workflow<I, O> = (ctx: RunContext, input: I) => Promise<O>;
|
|
58
|
+
export interface RunHandle<O> {
|
|
59
|
+
readonly runId: string;
|
|
60
|
+
result(): Promise<RunOutcome<O>>;
|
|
61
|
+
}
|
|
62
|
+
export type ExecutionState = "running" | "waiting" | "completed" | "failed" | "cancelled";
|
|
63
|
+
export interface ExecutionStatus {
|
|
64
|
+
state: ExecutionState;
|
|
65
|
+
}
|
|
66
|
+
export interface RecoveredRun<O = unknown> {
|
|
67
|
+
runId: string;
|
|
68
|
+
state: "completed" | "waiting" | "failed" | "cancelled";
|
|
69
|
+
error?: string;
|
|
70
|
+
output?: O;
|
|
71
|
+
}
|
|
72
|
+
export interface Engine {
|
|
73
|
+
/** Registers the run and begins executing it in the background. */
|
|
74
|
+
start<I, O>(runId: string, workflow: Workflow<I, O>, input: I): Promise<RunHandle<O>>;
|
|
75
|
+
/** Starts the run and waits until it completes or stops to wait. */
|
|
76
|
+
run<I, O>(runId: string, workflow: Workflow<I, O>, input: I): Promise<RunOutcome<O>>;
|
|
77
|
+
/** Continues an unfinished run from its recorded steps, whoever held it. */
|
|
78
|
+
resume<I, O>(runId: string, workflow: Workflow<I, O>): Promise<RunOutcome<O>>;
|
|
79
|
+
/**
|
|
80
|
+
* Delivers a decision to the wait named `name`. The first signal for a name wins; one that comes
|
|
81
|
+
* after the wait ended changes nothing. A run waiting in this process continues at once; any other
|
|
82
|
+
* continues when `recover` next runs.
|
|
83
|
+
*/
|
|
84
|
+
signal(runId: string, name: string, value: unknown): Promise<SignalResult>;
|
|
85
|
+
/**
|
|
86
|
+
* Continues every run whose worker died or let its lease run out, and every waiting run whose
|
|
87
|
+
* wait can now end. `resolve` picks the workflow for a run; return undefined to leave it alone.
|
|
88
|
+
*/
|
|
89
|
+
recover<I, O>(resolve: (runId: string, input: I) => Workflow<I, O> | undefined): Promise<RecoveredRun<O>[]>;
|
|
90
|
+
/**
|
|
91
|
+
* Cancels a run. A step that is running gets its abort signal fired; a waiting run ends at once
|
|
92
|
+
* where it holds the run, or when `recover` next runs. The workflow sees RunCancelled at its next
|
|
93
|
+
* step or wait and returns.
|
|
94
|
+
*/
|
|
95
|
+
cancel(runId: string): Promise<CancelResult>;
|
|
96
|
+
status(runId: string): Promise<ExecutionStatus | undefined>;
|
|
97
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/env.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Loads `<root>/.env` into `process.env` without replacing variables that are already set, and
|
|
3
|
+
* returns the file path, or `undefined` when the project has none. Values are never logged.
|
|
4
|
+
*/
|
|
5
|
+
export declare function loadProjectEnv(root: string): string | undefined;
|
|
6
|
+
/** The `.env` files this process loaded through {@link loadProjectEnv}. */
|
|
7
|
+
export declare function loadedEnvFiles(): string[];
|
package/dist/env.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
const loaded = new Set();
|
|
4
|
+
/**
|
|
5
|
+
* Loads `<root>/.env` into `process.env` without replacing variables that are already set, and
|
|
6
|
+
* returns the file path, or `undefined` when the project has none. Values are never logged.
|
|
7
|
+
*/
|
|
8
|
+
export function loadProjectEnv(root) {
|
|
9
|
+
const file = join(root, ".env");
|
|
10
|
+
if (!existsSync(file))
|
|
11
|
+
return undefined;
|
|
12
|
+
process.loadEnvFile(file);
|
|
13
|
+
loaded.add(file);
|
|
14
|
+
return file;
|
|
15
|
+
}
|
|
16
|
+
/** The `.env` files this process loaded through {@link loadProjectEnv}. */
|
|
17
|
+
export function loadedEnvFiles() {
|
|
18
|
+
return [...loaded];
|
|
19
|
+
}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { PlicoEventPayloads } from "@plico-ai/contracts";
|
|
2
|
+
export declare class RuntimeDiagnosticError extends Error {
|
|
3
|
+
readonly payload: PlicoEventPayloads["run.error"];
|
|
4
|
+
constructor(payload: PlicoEventPayloads["run.error"]);
|
|
5
|
+
}
|
|
6
|
+
export declare class RunWaitingError extends Error {
|
|
7
|
+
readonly output: string | undefined;
|
|
8
|
+
constructor(output: string | undefined);
|
|
9
|
+
}
|
|
10
|
+
export declare function errorMessage(error: unknown): string;
|
|
11
|
+
export declare function runtimeErrorPayload(error: unknown): PlicoEventPayloads["run.error"];
|
|
12
|
+
export type ModelErrorKind = "rate_limit" | "overloaded" | "auth" | "bad_request" | "context_length" | "aborted" | "network";
|
|
13
|
+
/** Provider adapters normalize errors to these provider-neutral kinds. */
|
|
14
|
+
export declare class ModelProviderError extends Error {
|
|
15
|
+
readonly kind: ModelErrorKind;
|
|
16
|
+
readonly retryable: boolean;
|
|
17
|
+
constructor(kind: ModelErrorKind, message: string);
|
|
18
|
+
}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export class RuntimeDiagnosticError extends Error {
|
|
2
|
+
payload;
|
|
3
|
+
constructor(payload) {
|
|
4
|
+
super(typeof payload.message === "string" ? payload.message : "Runtime diagnostic error");
|
|
5
|
+
this.payload = payload;
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
export class RunWaitingError extends Error {
|
|
9
|
+
output;
|
|
10
|
+
constructor(output) {
|
|
11
|
+
super("Run waiting for approval");
|
|
12
|
+
this.output = output;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
export function errorMessage(error) {
|
|
16
|
+
if (error instanceof Error && error.message) {
|
|
17
|
+
return error.message;
|
|
18
|
+
}
|
|
19
|
+
return "Unknown runtime error";
|
|
20
|
+
}
|
|
21
|
+
export function runtimeErrorPayload(error) {
|
|
22
|
+
if (error instanceof RuntimeDiagnosticError) {
|
|
23
|
+
return error.payload;
|
|
24
|
+
}
|
|
25
|
+
return {
|
|
26
|
+
phase: "runtime",
|
|
27
|
+
message: errorMessage(error),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/** Provider adapters normalize errors to these provider-neutral kinds. */
|
|
31
|
+
export class ModelProviderError extends Error {
|
|
32
|
+
kind;
|
|
33
|
+
retryable;
|
|
34
|
+
constructor(kind, message) {
|
|
35
|
+
super(message);
|
|
36
|
+
this.kind = kind;
|
|
37
|
+
this.name = "ModelProviderError";
|
|
38
|
+
this.retryable = kind === "rate_limit" || kind === "overloaded" || kind === "network";
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
const fixtureSuffix = ".script.json";
|
|
5
|
+
function toEvalCase(file, parsed) {
|
|
6
|
+
const fixture = (Array.isArray(parsed) ? { script: parsed } : parsed);
|
|
7
|
+
if (!Array.isArray(fixture?.script)) {
|
|
8
|
+
throw new Error(`Invalid eval fixture ${file}: expected a step array or a "script" array.`);
|
|
9
|
+
}
|
|
10
|
+
const entry = {
|
|
11
|
+
name: typeof fixture.name === "string" ? fixture.name : file.slice(0, -fixtureSuffix.length),
|
|
12
|
+
script: fixture.script,
|
|
13
|
+
};
|
|
14
|
+
if (typeof fixture.description === "string")
|
|
15
|
+
entry.description = fixture.description;
|
|
16
|
+
if (fixture.expect !== undefined)
|
|
17
|
+
entry.expect = fixture.expect;
|
|
18
|
+
return entry;
|
|
19
|
+
}
|
|
20
|
+
async function readFixtures(evalsDir) {
|
|
21
|
+
let entries;
|
|
22
|
+
try {
|
|
23
|
+
entries = await readdir(evalsDir);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return [];
|
|
27
|
+
}
|
|
28
|
+
const scriptFiles = entries.filter((e) => e.endsWith(fixtureSuffix)).sort();
|
|
29
|
+
return Promise.all(scriptFiles.map(async (file) => ({ file, raw: await readFile(join(evalsDir, file), "utf8") })));
|
|
30
|
+
}
|
|
31
|
+
function parseFixture(file, raw) {
|
|
32
|
+
try {
|
|
33
|
+
return JSON.parse(raw);
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
throw new Error(`Invalid eval fixture ${file}: ${error.message}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
export async function discoverEvalCases(evalsDir) {
|
|
40
|
+
const fixtures = await readFixtures(evalsDir);
|
|
41
|
+
return fixtures.map(({ file, raw }) => toEvalCase(file, parseFixture(file, raw)));
|
|
42
|
+
}
|
|
43
|
+
export async function digestEvalFixtures(evalsDir) {
|
|
44
|
+
const hash = createHash("sha256");
|
|
45
|
+
for (const { file, raw } of await readFixtures(evalsDir)) {
|
|
46
|
+
hash.update(`${file}\0${raw}\0`);
|
|
47
|
+
}
|
|
48
|
+
return hash.digest("hex");
|
|
49
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { PlicoEvent, RunStatus } from "@plico-ai/contracts";
|
|
2
|
+
import type { EvalCase } from "../types/index.js";
|
|
3
|
+
/**
|
|
4
|
+
* Turns a stored run into an eval case. Each tool call keeps the output it produced as the mocked
|
|
5
|
+
* `result`, so the case replays without running any handler. Files a tool wrote are not carried over.
|
|
6
|
+
*/
|
|
7
|
+
export declare function evalCaseFromRun(name: string, status: RunStatus, events: PlicoEvent[]): EvalCase;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turns a stored run into an eval case. Each tool call keeps the output it produced as the mocked
|
|
3
|
+
* `result`, so the case replays without running any handler. Files a tool wrote are not carried over.
|
|
4
|
+
*/
|
|
5
|
+
export function evalCaseFromRun(name, status, events) {
|
|
6
|
+
const script = [];
|
|
7
|
+
let output;
|
|
8
|
+
let pending;
|
|
9
|
+
for (const event of events) {
|
|
10
|
+
if (event.type === "assistant.output") {
|
|
11
|
+
script.push({ type: "assistant.output", content: event.payload.content });
|
|
12
|
+
output = event.payload.content;
|
|
13
|
+
}
|
|
14
|
+
else if (event.type === "tool.call") {
|
|
15
|
+
pending = {
|
|
16
|
+
type: "tool.call",
|
|
17
|
+
toolName: event.payload.toolName,
|
|
18
|
+
arguments: event.payload.arguments,
|
|
19
|
+
};
|
|
20
|
+
script.push(pending);
|
|
21
|
+
}
|
|
22
|
+
else if (event.type === "tool.result" && pending) {
|
|
23
|
+
pending.result = event.payload.result;
|
|
24
|
+
pending = undefined;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return {
|
|
28
|
+
name,
|
|
29
|
+
script,
|
|
30
|
+
expect: { status, ...(output === undefined ? {} : { outputContains: output }) },
|
|
31
|
+
};
|
|
32
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { getProjectPaths } from "@plico-ai/core";
|
|
2
|
+
import { runProject } from "../run/index.js";
|
|
3
|
+
import { digestEvalFixtures, discoverEvalCases } from "./discover.js";
|
|
4
|
+
export async function runEvals(projectRoot, options = {}) {
|
|
5
|
+
const evalsDir = getProjectPaths(projectRoot).directories.evals;
|
|
6
|
+
const cases = await discoverEvalCases(evalsDir);
|
|
7
|
+
const fixturesDigest = await digestEvalFixtures(evalsDir);
|
|
8
|
+
const results = [];
|
|
9
|
+
for (const evalCase of cases) {
|
|
10
|
+
const runId = `eval-${evalCase.name.replace(/[^a-z0-9_-]/gi, "_")}-${crypto.randomUUID().slice(0, 8)}`;
|
|
11
|
+
const runOpts = {
|
|
12
|
+
script: evalCase.script,
|
|
13
|
+
mockTools: true,
|
|
14
|
+
runId,
|
|
15
|
+
};
|
|
16
|
+
if (options.eventStore !== undefined)
|
|
17
|
+
runOpts.eventStore = options.eventStore;
|
|
18
|
+
const result = await runProject(projectRoot, runOpts);
|
|
19
|
+
const diagnostics = [];
|
|
20
|
+
let passed = true;
|
|
21
|
+
if (evalCase.expect?.status && result.status !== evalCase.expect.status) {
|
|
22
|
+
passed = false;
|
|
23
|
+
diagnostics.push(`Expected status "${evalCase.expect.status}" but got "${result.status}"`);
|
|
24
|
+
}
|
|
25
|
+
if (!evalCase.expect?.status && result.status !== "completed") {
|
|
26
|
+
passed = false;
|
|
27
|
+
diagnostics.push(`Run ended with status "${result.status}"`);
|
|
28
|
+
}
|
|
29
|
+
if (evalCase.expect?.outputContains) {
|
|
30
|
+
const output = result.output ?? "";
|
|
31
|
+
if (!output.includes(evalCase.expect.outputContains)) {
|
|
32
|
+
passed = false;
|
|
33
|
+
diagnostics.push(`Expected output to contain "${evalCase.expect.outputContains}" but got "${output}"`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
for (const event of result.events) {
|
|
37
|
+
if (event.type === "run.error" && typeof event.payload.message === "string")
|
|
38
|
+
diagnostics.push(event.payload.message);
|
|
39
|
+
}
|
|
40
|
+
if (result.errors?.length) {
|
|
41
|
+
for (const err of result.errors) {
|
|
42
|
+
diagnostics.push(`${err.path}: ${err.message}`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const entry = {
|
|
46
|
+
name: evalCase.name,
|
|
47
|
+
status: passed ? "pass" : "fail",
|
|
48
|
+
runId: result.runId,
|
|
49
|
+
runStatus: result.status,
|
|
50
|
+
};
|
|
51
|
+
if (evalCase.description !== undefined)
|
|
52
|
+
entry.description = evalCase.description;
|
|
53
|
+
if (result.output !== undefined)
|
|
54
|
+
entry.output = result.output;
|
|
55
|
+
if (diagnostics.length > 0)
|
|
56
|
+
entry.diagnostics = diagnostics;
|
|
57
|
+
results.push(entry);
|
|
58
|
+
}
|
|
59
|
+
const passed = results.filter((r) => r.status === "pass").length;
|
|
60
|
+
const failed = results.filter((r) => r.status === "fail").length;
|
|
61
|
+
const summary = { total: results.length, passed, failed, results };
|
|
62
|
+
await options.eventStore?.persistEvalReport({
|
|
63
|
+
schemaVersion: 1,
|
|
64
|
+
...summary,
|
|
65
|
+
evaluatedAt: new Date().toISOString(),
|
|
66
|
+
fixturesDigest,
|
|
67
|
+
});
|
|
68
|
+
return summary;
|
|
69
|
+
}
|
package/dist/events.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { EventStreamType, PlicoEventPayloads } from "@plico-ai/contracts";
|
|
2
|
+
import type { EventStore, PlicoEvent, PlicoEventType } from "./types/index.js";
|
|
3
|
+
export declare const runStreamType: EventStreamType;
|
|
4
|
+
/**
|
|
5
|
+
* Build the terminal failure pair both recovery paths append. The caller supplies the tail so
|
|
6
|
+
* appending to an event store and inserting directly into SQLite share one sequencing scheme.
|
|
7
|
+
*/
|
|
8
|
+
export declare function runFailureEvents(runId: string, last: {
|
|
9
|
+
sequence: number;
|
|
10
|
+
type: string;
|
|
11
|
+
} | undefined, message: string, timestamp?: string): PlicoEvent[];
|
|
12
|
+
export declare class InMemoryEventLog {
|
|
13
|
+
#private;
|
|
14
|
+
private readonly runId;
|
|
15
|
+
private readonly store;
|
|
16
|
+
/**
|
|
17
|
+
* Opens the log of a run that may have started before. The loop emits the same events again when
|
|
18
|
+
* it replays; events already stored are matched, not written twice.
|
|
19
|
+
*/
|
|
20
|
+
static open(runId: string, store: EventStore | undefined): Promise<InMemoryEventLog>;
|
|
21
|
+
constructor(runId: string, store?: EventStore | undefined);
|
|
22
|
+
append<T extends PlicoEventType>(type: T, payload: PlicoEventPayloads[T]): Promise<PlicoEvent>;
|
|
23
|
+
/** Skip previously persisted live chunks when replaying a model step. */
|
|
24
|
+
skipModelDeltas(turn: number): void;
|
|
25
|
+
/** Continue after the recorded tail when recovery cannot replay the workflow. */
|
|
26
|
+
continueAfterReplay(): void;
|
|
27
|
+
all(): PlicoEvent[];
|
|
28
|
+
}
|
package/dist/events.js
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { ProjectChangedError } from "./engine/index.js";
|
|
2
|
+
export const runStreamType = "run";
|
|
3
|
+
/**
|
|
4
|
+
* Build the terminal failure pair both recovery paths append. The caller supplies the tail so
|
|
5
|
+
* appending to an event store and inserting directly into SQLite share one sequencing scheme.
|
|
6
|
+
*/
|
|
7
|
+
export function runFailureEvents(runId, last, message, timestamp = new Date().toISOString()) {
|
|
8
|
+
const events = [];
|
|
9
|
+
let sequence = last?.sequence ?? 0;
|
|
10
|
+
if (last?.type !== "run.error") {
|
|
11
|
+
sequence += 1;
|
|
12
|
+
events.push({
|
|
13
|
+
schemaVersion: 1,
|
|
14
|
+
streamType: "run",
|
|
15
|
+
streamId: runId,
|
|
16
|
+
runId,
|
|
17
|
+
id: `evt-${String(sequence).padStart(4, "0")}`,
|
|
18
|
+
sequence,
|
|
19
|
+
timestamp,
|
|
20
|
+
type: "run.error",
|
|
21
|
+
payload: { phase: "runtime", message },
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
sequence += 1;
|
|
25
|
+
events.push({
|
|
26
|
+
schemaVersion: 1,
|
|
27
|
+
streamType: "run",
|
|
28
|
+
streamId: runId,
|
|
29
|
+
runId,
|
|
30
|
+
id: `evt-${String(sequence).padStart(4, "0")}`,
|
|
31
|
+
sequence,
|
|
32
|
+
timestamp,
|
|
33
|
+
type: "run.failed",
|
|
34
|
+
payload: { phase: "runtime" },
|
|
35
|
+
});
|
|
36
|
+
return events;
|
|
37
|
+
}
|
|
38
|
+
export class InMemoryEventLog {
|
|
39
|
+
runId;
|
|
40
|
+
store;
|
|
41
|
+
#events = [];
|
|
42
|
+
#replayed = 0;
|
|
43
|
+
#cursor = 0;
|
|
44
|
+
/**
|
|
45
|
+
* Opens the log of a run that may have started before. The loop emits the same events again when
|
|
46
|
+
* it replays; events already stored are matched, not written twice.
|
|
47
|
+
*/
|
|
48
|
+
static async open(runId, store) {
|
|
49
|
+
const log = new InMemoryEventLog(runId, store);
|
|
50
|
+
if (store)
|
|
51
|
+
log.#events.push(...(await store.getEvents(runId)));
|
|
52
|
+
log.#replayed = log.#events.length;
|
|
53
|
+
return log;
|
|
54
|
+
}
|
|
55
|
+
constructor(runId, store = undefined) {
|
|
56
|
+
this.runId = runId;
|
|
57
|
+
this.store = store;
|
|
58
|
+
}
|
|
59
|
+
async append(type, payload) {
|
|
60
|
+
if (this.#cursor < this.#replayed) {
|
|
61
|
+
const recorded = this.#events[this.#cursor];
|
|
62
|
+
if (recorded.type !== type || JSON.stringify(recorded.payload) !== JSON.stringify(payload)) {
|
|
63
|
+
// Later events (the failure) are appended after what the earlier attempt stored.
|
|
64
|
+
this.#cursor = this.#replayed;
|
|
65
|
+
throw new ProjectChangedError("event", this.#cursor, recorded.type, type);
|
|
66
|
+
}
|
|
67
|
+
this.#cursor += 1;
|
|
68
|
+
return recorded;
|
|
69
|
+
}
|
|
70
|
+
const sequence = this.#events.length + 1;
|
|
71
|
+
// TypeScript cannot prove this generic object is a union member; callers are checked against PlicoEventPayloads[T].
|
|
72
|
+
const event = {
|
|
73
|
+
schemaVersion: 1,
|
|
74
|
+
streamType: runStreamType,
|
|
75
|
+
streamId: this.runId,
|
|
76
|
+
runId: this.runId,
|
|
77
|
+
id: `evt-${String(sequence).padStart(4, "0")}`,
|
|
78
|
+
sequence,
|
|
79
|
+
timestamp: new Date().toISOString(),
|
|
80
|
+
type,
|
|
81
|
+
payload,
|
|
82
|
+
};
|
|
83
|
+
await this.store?.appendEvent(event);
|
|
84
|
+
this.#events.push(event);
|
|
85
|
+
return event;
|
|
86
|
+
}
|
|
87
|
+
/** Skip previously persisted live chunks when replaying a model step. */
|
|
88
|
+
skipModelDeltas(turn) {
|
|
89
|
+
while (this.#cursor < this.#replayed) {
|
|
90
|
+
const event = this.#events[this.#cursor];
|
|
91
|
+
if ((event?.type !== "model.delta" && event?.type !== "model.attempt") ||
|
|
92
|
+
event.payload.turn !== turn)
|
|
93
|
+
break;
|
|
94
|
+
this.#cursor++;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/** Continue after the recorded tail when recovery cannot replay the workflow. */
|
|
98
|
+
continueAfterReplay() {
|
|
99
|
+
this.#cursor = this.#replayed;
|
|
100
|
+
}
|
|
101
|
+
all() {
|
|
102
|
+
return [...this.#events];
|
|
103
|
+
}
|
|
104
|
+
}
|
package/dist/guards.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function isRecord(value: unknown): value is Record<string, unknown>;
|
package/dist/guards.js
ADDED
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export { buildArtifactWorkspace, readArtifactSnapshot, readArtifactVersion } from "./artifacts.js";
|
|
2
|
+
export { createEngine, createMemoryJournal, createSqliteEngine, type Engine, type Journal, ProjectChangedError, type RecoveredRun, type RunContext, type Workflow, } from "./engine/index.js";
|
|
3
|
+
export { loadedEnvFiles, loadProjectEnv } from "./env.js";
|
|
4
|
+
export { type ModelErrorKind, ModelProviderError } from "./errors.js";
|
|
5
|
+
export { evalCaseFromRun, runEvals } from "./evals/index.js";
|
|
6
|
+
export { runFailureEvents } from "./events.js";
|
|
7
|
+
export { composeInstructions } from "./instructions.js";
|
|
8
|
+
export { createConfiguredModelProvider, modelProviderAccountKey, modelProviderRecoveryKey, supportsModelProvider, } from "./providers/configured.js";
|
|
9
|
+
export { createOpenAICompatibleProvider, type OpenAICompatibleProviderOptions, } from "./providers/openai-compatible.js";
|
|
10
|
+
export { checkReadiness } from "./readiness/index.js";
|
|
11
|
+
export { recoverRuns, runProject } from "./run/index.js";
|
|
12
|
+
export { normalizeScript, ScriptValidationError } from "./script.js";
|
|
13
|
+
export { createSqliteEventStore } from "./storage/index.js";
|
|
14
|
+
export { matchesMetadata } from "./storage/metadata.js";
|
|
15
|
+
export { createSqliteThreadStore, ThreadBusyError, type ThreadStore, } from "./storage/sqlite/threads.js";
|
|
16
|
+
export { createThreadTitler, type ThreadTitler } from "./threads/titler.js";
|
|
17
|
+
export { discoverTools } from "./tools/index.js";
|
|
18
|
+
export type { ArtifactContent, ArtifactFile, ArtifactFileInput, ArtifactFileMetadata, ArtifactSnapshot, ArtifactSnapshotDetail, ArtifactVersion, ArtifactWorkspace, ComposedInstructions, CreateSqliteEventStoreOptions, DeclaredTool, DiagnosticSeverity, DryRunScriptStep, EvalCase, EvalCaseResult, EvalExpectation, EvalReport, EvalSummary, EventStore, JsonSchema, ModelChunk, ModelProvider, ModelRequest, ModelResponse, ModelToolCall, PlicoEvent, PlicoEventType, ProviderMessage, ReadinessDiagnostic, ReadinessOptions, ReadinessReport, RunEvalsOptions, RunProjectOptions, RunResult, RunStatus, RunSummary, ToolApprovalPolicy, ToolArtifactContext, ToolArtifactFile, ToolDefinition, ToolHandlerContext, ToolInputParseResult, ToolInputSchema, ZodInputParseResult, ZodInputSchema, } from "./types/index.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export { buildArtifactWorkspace, readArtifactSnapshot, readArtifactVersion } from "./artifacts.js";
|
|
2
|
+
export { createEngine, createMemoryJournal, createSqliteEngine, ProjectChangedError, } from "./engine/index.js";
|
|
3
|
+
export { loadedEnvFiles, loadProjectEnv } from "./env.js";
|
|
4
|
+
export { ModelProviderError } from "./errors.js";
|
|
5
|
+
export { evalCaseFromRun, runEvals } from "./evals/index.js";
|
|
6
|
+
export { runFailureEvents } from "./events.js";
|
|
7
|
+
export { composeInstructions } from "./instructions.js";
|
|
8
|
+
export { createConfiguredModelProvider, modelProviderAccountKey, modelProviderRecoveryKey, supportsModelProvider, } from "./providers/configured.js";
|
|
9
|
+
export { createOpenAICompatibleProvider, } from "./providers/openai-compatible.js";
|
|
10
|
+
export { checkReadiness } from "./readiness/index.js";
|
|
11
|
+
export { recoverRuns, runProject } from "./run/index.js";
|
|
12
|
+
export { normalizeScript, ScriptValidationError } from "./script.js";
|
|
13
|
+
export { createSqliteEventStore } from "./storage/index.js";
|
|
14
|
+
export { matchesMetadata } from "./storage/metadata.js";
|
|
15
|
+
export { createSqliteThreadStore, ThreadBusyError, } from "./storage/sqlite/threads.js";
|
|
16
|
+
export { createThreadTitler } from "./threads/titler.js";
|
|
17
|
+
export { discoverTools } from "./tools/index.js";
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type LoadedProject, type Skill } from "@plico-ai/core";
|
|
2
|
+
import type { ComposedInstructions } from "./types/index.js";
|
|
3
|
+
/** The agent's instructions plus a list of skills. A skill's full text loads on demand. */
|
|
4
|
+
export declare function composeInstructions(project: LoadedProject, loaded?: Skill[]): Promise<ComposedInstructions>;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { getProjectPaths, getProjectRelativePaths, parseAgentFile, readSkills, SKILL_TOOL_NAME, } from "@plico-ai/core";
|
|
3
|
+
/** The agent's instructions plus a list of skills. A skill's full text loads on demand. */
|
|
4
|
+
export async function composeInstructions(project, loaded) {
|
|
5
|
+
const projectPaths = getProjectPaths(project.root);
|
|
6
|
+
const sources = [getProjectRelativePaths().agentFile];
|
|
7
|
+
const parts = [parseAgentFile(await readFile(projectPaths.agentFile, "utf8")).body];
|
|
8
|
+
const skills = loaded ?? (await readSkills(project.root)).skills;
|
|
9
|
+
if (skills.length > 0) {
|
|
10
|
+
sources.push(...skills.map((skill) => skill.source));
|
|
11
|
+
parts.push([
|
|
12
|
+
"## Skills",
|
|
13
|
+
"",
|
|
14
|
+
`Call ${SKILL_TOOL_NAME} with a skill's name to read its full instructions before you use it.`,
|
|
15
|
+
"",
|
|
16
|
+
...skills.map((skill) => `- ${skill.name}: ${skill.description}`),
|
|
17
|
+
].join("\n"));
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
sources,
|
|
21
|
+
content: parts.map((part) => part.trimEnd()).join("\n\n"),
|
|
22
|
+
};
|
|
23
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ProjectConfig } from "@plico-ai/contracts";
|
|
2
|
+
import type { DeclaredTool } from "../types/tools.js";
|
|
3
|
+
export type ApprovalDecision = "none" | "required";
|
|
4
|
+
export declare function decideApproval(tool: DeclaredTool, config: ProjectConfig): {
|
|
5
|
+
required: boolean;
|
|
6
|
+
reason?: string;
|
|
7
|
+
};
|
|
8
|
+
export declare function approvalTimeoutMs(config: ProjectConfig): number;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export function decideApproval(tool, config) {
|
|
2
|
+
const policy = config.approvals;
|
|
3
|
+
const override = policy?.tools?.[tool.name];
|
|
4
|
+
let decision;
|
|
5
|
+
if (override === "required" || override === "none") {
|
|
6
|
+
decision = override;
|
|
7
|
+
}
|
|
8
|
+
else if (tool.approval.required) {
|
|
9
|
+
decision = "required";
|
|
10
|
+
}
|
|
11
|
+
else if (policy?.default === "required") {
|
|
12
|
+
decision = "required";
|
|
13
|
+
}
|
|
14
|
+
else {
|
|
15
|
+
decision = "none";
|
|
16
|
+
}
|
|
17
|
+
// Config can escalate but never silently drop a tool-declared requirement.
|
|
18
|
+
if (tool.approval.required && decision === "none") {
|
|
19
|
+
decision = "required";
|
|
20
|
+
}
|
|
21
|
+
return tool.approval.reason
|
|
22
|
+
? { required: decision === "required", reason: tool.approval.reason }
|
|
23
|
+
: { required: decision === "required" };
|
|
24
|
+
}
|
|
25
|
+
export function approvalTimeoutMs(config) {
|
|
26
|
+
return config.approvals?.timeoutMs ?? 30 * 60 * 1000;
|
|
27
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type LanguageModel } from "ai";
|
|
2
|
+
import type { ModelProvider } from "../types/index.js";
|
|
3
|
+
export interface AiSdkProviderOptions {
|
|
4
|
+
/** The variable that holds the credential. Without one, the model is built with no key. */
|
|
5
|
+
apiKeyEnv?: string | undefined;
|
|
6
|
+
/** Builds the AI SDK model once the credential is known. */
|
|
7
|
+
createModel: (apiKey: string | undefined) => LanguageModel | Promise<LanguageModel>;
|
|
8
|
+
}
|
|
9
|
+
/** Wraps an AI SDK language model behind the Plico model port. Provider retries are off. */
|
|
10
|
+
export declare function createAiSdkModelProvider(options: AiSdkProviderOptions): ModelProvider;
|