@nirmos/sdk 0.3.3 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +24 -1
  2. package/dist/cjs/core/version.js +1 -1
  3. package/dist/cjs/evaluations/types.js +3 -0
  4. package/dist/cjs/evaluations/types.js.map +1 -0
  5. package/dist/cjs/memory/execution-types.js +3 -0
  6. package/dist/cjs/memory/execution-types.js.map +1 -0
  7. package/dist/esm/core/version.d.ts +1 -1
  8. package/dist/esm/core/version.js +1 -1
  9. package/dist/esm/evaluations/types.d.ts +13 -0
  10. package/dist/esm/evaluations/types.d.ts.map +1 -0
  11. package/dist/esm/evaluations/types.js +2 -0
  12. package/dist/esm/evaluations/types.js.map +1 -0
  13. package/dist/esm/index.d.ts +1 -0
  14. package/dist/esm/index.d.ts.map +1 -1
  15. package/dist/esm/management/contracts.d.ts +4811 -9037
  16. package/dist/esm/management/contracts.d.ts.map +1 -1
  17. package/dist/esm/memory/execution-types.d.ts +35 -0
  18. package/dist/esm/memory/execution-types.d.ts.map +1 -0
  19. package/dist/esm/memory/execution-types.js +2 -0
  20. package/dist/esm/memory/execution-types.js.map +1 -0
  21. package/dist/esm/memory/types.d.ts +2 -25
  22. package/dist/esm/memory/types.d.ts.map +1 -1
  23. package/examples/evaluation-offline.mjs +151 -0
  24. package/examples/evaluation-quickstart.mjs +210 -0
  25. package/examples/evaluation-quickstart.ts +255 -0
  26. package/examples/evaluation-trace-import.mjs +81 -0
  27. package/examples/evaluation-trace-import.ts +105 -0
  28. package/examples/tsconfig.json +1 -1
  29. package/package.json +1 -1
  30. package/src/core/version.ts +1 -1
  31. package/src/evaluations/types.ts +18 -0
  32. package/src/index.ts +11 -0
  33. package/src/management/contracts.ts +30 -67
  34. package/src/memory/execution-types.ts +2 -0
  35. package/src/memory/types.ts +2 -25
@@ -0,0 +1,35 @@
1
+ export type MemoryIngestResponse = {
2
+ "memoryRunId": string;
3
+ "memories": Array<{
4
+ "id": string;
5
+ "content": string;
6
+ "type": "preference" | "fact" | "relationship" | "decision" | "constraint" | "commitment" | "procedure" | "event" | "project_fact" | "temporary_state" | "observation" | "custom";
7
+ "layer": "atomic" | "observation";
8
+ "status": "active" | "superseded" | "expired" | "archived" | "deleted";
9
+ "validFrom": string | null;
10
+ "validUntil": string | null;
11
+ "confidence": number | null;
12
+ }>;
13
+ "candidateCount": number;
14
+ "acceptedCount": number;
15
+ "rejectedCount": number;
16
+ "insertedCount": number;
17
+ "mergedCount": number;
18
+ "reinforcedCount": number;
19
+ "supersededCount": number;
20
+ "ignoredCount": number;
21
+ "indexing": {
22
+ "indexed": number;
23
+ "failed": number;
24
+ "diagnostics"?: Array<{
25
+ "code": string;
26
+ "status": number;
27
+ "retryable": boolean;
28
+ "details"?: {
29
+ [key: string]: unknown;
30
+ };
31
+ }>;
32
+ };
33
+ "replayed": boolean;
34
+ };
35
+ //# sourceMappingURL=execution-types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"execution-types.d.ts","sourceRoot":"","sources":["../../../src/memory/execution-types.ts"],"names":[],"mappings":"AACA,MAAM,MAAM,oBAAoB,GAAG;IAAE,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,GAAG,MAAM,GAAG,cAAc,GAAG,UAAU,GAAG,YAAY,GAAG,YAAY,GAAG,WAAW,GAAG,OAAO,GAAG,cAAc,GAAG,iBAAiB,GAAG,aAAa,GAAG,QAAQ,CAAC;QAAC,OAAO,EAAE,QAAQ,GAAG,aAAa,CAAC;QAAC,QAAQ,EAAE,QAAQ,GAAG,YAAY,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAAE,CAAC,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAC;YAAC,WAAW,EAAE,OAAO,CAAC;YAAC,SAAS,CAAC,EAAE;gBAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;aAAE,CAAC;SAAE,CAAC,CAAC;KAAE,CAAC;IAAC,UAAU,EAAE,OAAO,CAAC;CAAE,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=execution-types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"execution-types.js","sourceRoot":"","sources":["../../../src/memory/execution-types.ts"],"names":[],"mappings":""}
@@ -84,31 +84,8 @@ export interface MemorySearchResponse {
84
84
  relevance: "relevant" | "raw";
85
85
  }
86
86
  export type MemoryContextResponse = MemorySearchResponse;
87
- export interface MemoryIndexingResult {
88
- /** Safe stage/provider diagnostics for partial indexing failures. */
89
- diagnostics?: Array<{
90
- code: string;
91
- status: number;
92
- retryable: boolean;
93
- details?: Record<string, unknown>;
94
- }>;
95
- indexed: number;
96
- failed: number;
97
- }
98
- export interface MemoryIngestResponse {
99
- memoryRunId: string;
100
- memories: MemoryResult[];
101
- candidateCount: number;
102
- acceptedCount: number;
103
- rejectedCount: number;
104
- insertedCount: number;
105
- mergedCount: number;
106
- reinforcedCount: number;
107
- supersededCount: number;
108
- ignoredCount: number;
109
- indexing: MemoryIndexingResult;
110
- replayed?: boolean;
111
- }
87
+ export type { MemoryIngestResponse } from "./execution-types.js";
88
+ export type MemoryIndexingResult = import("./execution-types.js").MemoryIngestResponse["indexing"];
112
89
  export interface MemoryManualRequest {
113
90
  memorySpaceId: string;
114
91
  content: string;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/memory/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AACD,MAAM,MAAM,WAAW,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAAC;AACtD,MAAM,MAAM,YAAY,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;AACxD,MAAM,MAAM,UAAU,GAAG,SAAS,CAAC,eAAe,CAAC,CAAC;AACpD,MAAM,MAAM,cAAc,GACxB,SAAS,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AACvD,MAAM,MAAM,cAAc,GACxB,SAAS,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AACxD,MAAM,MAAM,eAAe,GACzB,SAAS,CAAC,0BAA0B,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AACzD,MAAM,MAAM,gBAAgB,GAC1B,SAAS,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AACvD,MAAM,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACtD,MAAM,MAAM,aAAa,GAAG,SAAS,CAAC,uBAAuB,CAAC,CAAC;AAC/D,MAAM,MAAM,SAAS,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;AAClD,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;AAC9D,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;AAC9D,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;AAChE,MAAM,MAAM,gBAAgB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC;AAC5D,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAC7D,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAChE,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAC3D,MAAM,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;AACzD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;AAChE,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AACpE,MAAM,MAAM,WAAW,GAAG,WAAW,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;AAClE,MAAM,MAAM,gBAAgB,GACxB,cAAc,GACd,cAAc,GACd,KAAK,GACL,MAAM,GACN,OAAO,GACP,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,OAAO,CAAC;AACZ,MAAM,WAAW,aAAa;IAC5B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,GAAG,QAAQ,GAAG,WAAW,CAAC;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AACD,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,gBAAgB,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,SAAS,CAAC;CACtB;AACD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,aAAa,EAAE,CAAC;IAC3B,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,SAAS,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AACD,MAAM,WAAW,mBAAmB;IAClC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,WAAW,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;CAChE;AACD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,SAAS,CAAC,EAAE,UAAU,GAAG,KAAK,CAAC;IAC/B,iGAAiG;IACjG,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,mBAAmB,CAAC;CAC/B;AACD,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAChD,mBAAmB,EACnB,OAAO,CACR;IACC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;CAC1D;AACD,MAAM,MAAM,YAAY,GAAG,IAAI,CAC7B,UAAU,EACR,IAAI,GACJ,SAAS,GACT,MAAM,GACN,OAAO,GACP,QAAQ,GACR,WAAW,GACX,YAAY,GACZ,YAAY,CACf,CAAC;AACF,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,KAAK,CAAC,YAAY,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3E,SAAS,EAAE,UAAU,GAAG,KAAK,CAAC;CAC/B;AACD,MAAM,MAAM,qBAAqB,GAAG,oBAAoB,CAAC;AAEzD,MAAM,WAAW,oBAAoB;IACnC,qEAAqE;IACrE,WAAW,CAAC,EAAE,KAAK,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,OAAO,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AACD,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AACD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AACD,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,gFAAgF;IAChF,uBAAuB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,cAAc,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,QAAQ,CAAC;CAC9C;AACD,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC;AACD,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,MAAM,CAAC;CACvB;AACD,MAAM,WAAW,6BAA6B;IAC5C,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,aAAa,CAAC;CACxB;AACD,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAC3E,QAAQ,EAAE,oBAAoB,CAAC;CAChC"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/memory/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,KAAK,EAAE,CAAC,EAAE,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AACD,MAAM,MAAM,WAAW,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAAC;AACtD,MAAM,MAAM,YAAY,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;AACxD,MAAM,MAAM,UAAU,GAAG,SAAS,CAAC,eAAe,CAAC,CAAC;AACpD,MAAM,MAAM,cAAc,GACxB,SAAS,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AACvD,MAAM,MAAM,cAAc,GACxB,SAAS,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AACxD,MAAM,MAAM,eAAe,GACzB,SAAS,CAAC,0BAA0B,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AACzD,MAAM,MAAM,gBAAgB,GAC1B,SAAS,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AACvD,MAAM,MAAM,YAAY,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACtD,MAAM,MAAM,aAAa,GAAG,SAAS,CAAC,uBAAuB,CAAC,CAAC;AAC/D,MAAM,MAAM,SAAS,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;AAClD,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;AAC9D,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC,mBAAmB,CAAC,CAAC;AAC9D,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;AAChE,MAAM,MAAM,gBAAgB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC;AAC5D,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAC7D,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAChE,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAC3D,MAAM,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;AACzD,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;AAChE,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AACpE,MAAM,MAAM,WAAW,GAAG,WAAW,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;AAClE,MAAM,MAAM,gBAAgB,GACxB,cAAc,GACd,cAAc,GACd,KAAK,GACL,MAAM,GACN,OAAO,GACP,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,OAAO,CAAC;AACZ,MAAM,WAAW,aAAa;IAC5B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,GAAG,QAAQ,GAAG,WAAW,CAAC;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AACD,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,gBAAgB,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,SAAS,CAAC;CACtB;AACD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,aAAa,EAAE,CAAC;IAC3B,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,aAAa,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,SAAS,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AACD,MAAM,WAAW,mBAAmB;IAClC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;IACvB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,WAAW,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;CAChE;AACD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,mGAAmG;IACnG,SAAS,CAAC,EAAE,UAAU,GAAG,KAAK,CAAC;IAC/B,iGAAiG;IACjG,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,mBAAmB,CAAC;CAC/B;AACD,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAChD,mBAAmB,EACnB,OAAO,CACR;IACC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;CAC1D;AACD,MAAM,MAAM,YAAY,GAAG,IAAI,CAC7B,UAAU,EACR,IAAI,GACJ,SAAS,GACT,MAAM,GACN,OAAO,GACP,QAAQ,GACR,WAAW,GACX,YAAY,GACZ,YAAY,CACf,CAAC;AACF,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,KAAK,CAAC,YAAY,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC3E,SAAS,EAAE,UAAU,GAAG,KAAK,CAAC;CAC/B;AACD,MAAM,MAAM,qBAAqB,GAAG,oBAAoB,CAAC;AAEzD,YAAY,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,OAAO,sBAAsB,EAAE,oBAAoB,CAAC,UAAU,CAAC,CAAC;AACnG,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AACD,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,gFAAgF;IAChF,uBAAuB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,cAAc,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,QAAQ,CAAC;CAC9C;AACD,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,oBAAoB,CAAC;CACjC;AACD,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,MAAM,CAAC;CACvB;AACD,MAAM,WAAW,6BAA6B;IAC5C,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,aAAa,CAAC;CACxB;AACD,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;IAC3E,QAAQ,EAAE,oBAAoB,CAAC;CAChC"}
@@ -0,0 +1,151 @@
1
+ // Deterministic SDK workflow fixture. No network, credentials, database or provider calls.
2
+ // Run: node packages/sdk/examples/evaluation-offline.mjs (after pnpm --filter @nirmos/sdk build).
3
+ // The live workflow uses the same public SDK function; this transport simulates service responses.
4
+ import assert from "node:assert/strict";
5
+ import { Nirmos } from "@nirmos/sdk";
6
+ if (process.env.NIRMOS_RUN_EVALUATION_EXAMPLE === "1") {
7
+ throw new Error("Unset NIRMOS_RUN_EVALUATION_EXAMPLE before running the offline fixture.");
8
+ }
9
+ const { evaluationQuickstart } = await import("./evaluation-quickstart.mjs");
10
+
11
+ const cases = [];
12
+ const archived = [];
13
+ let frozen = false;
14
+ let polls = 0;
15
+ let bindings;
16
+ const json = (data) =>
17
+ new Response(JSON.stringify({ data }), {
18
+ headers: { "content-type": "application/json" },
19
+ });
20
+ const sdk = new Nirmos({
21
+ auth: { type: "api-key", key: "offline-fixture" },
22
+ apiBaseUrl: "https://api.example.test",
23
+ runtimeBaseUrl: "https://runtime.example.test",
24
+ maxRetries: 0,
25
+ fetch: async (url, init) => {
26
+ const path = new URL(url).pathname;
27
+ const body = init.body ? JSON.parse(init.body) : {};
28
+ if (path.endsWith("/archive")) {
29
+ archived.push(path);
30
+ return json({ id: "archived" });
31
+ }
32
+ if (path === "/v1/evaluations/datasets")
33
+ return json({
34
+ dataset: { id: "dataset", name: body.name },
35
+ initialVersion: { id: "dataset-v1", version: 1, status: "draft" },
36
+ });
37
+ if (path === "/v1/evaluations/datasets/dataset/columns")
38
+ return json({ id: body.key });
39
+ if (path === "/v1/evaluations/datasets/dataset/cases") {
40
+ assert.equal(frozen, false);
41
+ cases.push(body.data);
42
+ return json({ id: `case-${cases.length}` });
43
+ }
44
+ if (path.endsWith("/freeze")) {
45
+ frozen = true;
46
+ return json({ id: "dataset-v1" });
47
+ }
48
+ if (path === "/v1/evaluations/suites") {
49
+ assert.ok(frozen);
50
+ return json({ id: "suite" });
51
+ }
52
+ if (path === "/v1/evaluations/columns") {
53
+ if (body.category === "target") {
54
+ assert.deepEqual(body.target.inputMapping, {
55
+ user_message: "dataset.question",
56
+ });
57
+ return json({
58
+ target: { id: "target" },
59
+ targetVersion: {
60
+ id: "target-v1",
61
+ version: 1,
62
+ columns: [{ id: "target-column" }],
63
+ },
64
+ column: { id: "target-column" },
65
+ });
66
+ }
67
+ bindings = body.config.inputMapping;
68
+ return json({ id: "evaluator-column", key: body.key, position: 1 });
69
+ }
70
+ if (path === "/v1/evaluations/evaluators") {
71
+ assert.equal(body.version.type, "exact_match");
72
+ assert.deepEqual(body.version.config, {
73
+ caseSensitive: true,
74
+ trimWhitespace: true,
75
+ normalizeWhitespace: false,
76
+ });
77
+ return json({
78
+ evaluator: { id: "evaluator", name: body.name },
79
+ version: { id: "evaluator-v1", version: 1 },
80
+ });
81
+ }
82
+ if (path === "/v1/evaluations/experiments/prepare") {
83
+ assert.deepEqual(body.targetVersionIds, ["target-v1"]);
84
+ return json({ id: "experiment" });
85
+ }
86
+ if (path === "/v1/evaluations/runs") {
87
+ assert.deepEqual(body.targetIds, ["target-v1"]);
88
+ return new Response(JSON.stringify({ id: "run", status: "QUEUED" }), {
89
+ status: 202,
90
+ });
91
+ }
92
+ if (path === "/v1/evaluations/runs/run")
93
+ return new Response(
94
+ JSON.stringify({
95
+ id: "run",
96
+ status: ++polls === 1 ? "RUNNING" : "COMPLETED",
97
+ summary: polls === 1 ? null : { overallPassRate: 0.5, byEvaluator: { exact_match: { count: 2, passRate: 0.5, average: null } } },
98
+ }),
99
+ );
100
+ if (path === "/v1/evaluations/runs/run/results") {
101
+ assert.deepEqual(bindings, {
102
+ actual: { source: "target", path: "output" },
103
+ expected: { source: "dataset", key: "expected" },
104
+ });
105
+ // Fixed target outputs deliberately demonstrate both passing and failing scores.
106
+ return new Response(
107
+ JSON.stringify({
108
+ data: cases.map((row, i) => {
109
+ const output = ["Paris", "Kyoto"][i];
110
+ const passed = output === row.expected;
111
+ return {
112
+ id: `trial-${i}`,
113
+ caseId: `case-${i}`,
114
+ status: "COMPLETED",
115
+ output,
116
+ error: null,
117
+ scores: [
118
+ {
119
+ evaluatorVersionId: "evaluator-v1",
120
+ status: "COMPLETED",
121
+ value: passed,
122
+ passed,
123
+ error: null,
124
+ },
125
+ ],
126
+ };
127
+ }),
128
+ total: cases.length,
129
+ page: 1,
130
+ pageSize: 25,
131
+ }),
132
+ );
133
+ }
134
+ throw new Error(`Unexpected fixture request: ${init.method} ${path}`);
135
+ },
136
+ });
137
+ const result = await evaluationQuickstart(sdk, {
138
+ providerId: "fixture-provider",
139
+ modelId: "fixture-model",
140
+ cleanup: true,
141
+ pollMs: 0,
142
+ });
143
+ assert.deepEqual(
144
+ result.results.data.map((trial) => trial.scores[0].passed),
145
+ [true, false],
146
+ );
147
+ assert.equal(result.summary.overallPassRate, 0.5);
148
+ assert.equal(archived.length, 3);
149
+ console.log(
150
+ "Offline workflow verified: two cases, pinned versions, mappings, polling, true/false scores and archive.",
151
+ );
@@ -0,0 +1,210 @@
1
+ // Generated from evaluation-quickstart.ts. Do not edit.
2
+ import { Nirmos } from "@nirmos/sdk";
3
+ function requireId(value, label) {
4
+ if (typeof value !== "string" || !value)
5
+ throw new Error(`Missing ${label}`);
6
+ return value;
7
+ }
8
+ export async function evaluationQuickstart(client, options) {
9
+ let datasetId;
10
+ let suiteId;
11
+ let evaluatorId;
12
+ let runId;
13
+ let terminal = false;
14
+ let submissionStarted = false;
15
+ try {
16
+ const dataset = await client.evaluations.datasets.create({
17
+ name: "SDK two-case demo",
18
+ });
19
+ datasetId = requireId(dataset.dataset.id, "dataset ID");
20
+ const datasetVersionId = requireId(dataset.initialVersion?.id, "dataset version ID");
21
+ console.log({ datasetId, datasetVersionId });
22
+ for (const column of [
23
+ { key: "question", name: "Question", role: "input" },
24
+ { key: "expected", name: "Expected", role: "expected" },
25
+ { key: "topic", name: "Topic", role: "metadata" },
26
+ ]) {
27
+ await client.evaluations.datasets.columns.create(datasetId, {
28
+ datasetVersionId,
29
+ ...column,
30
+ dataType: "text",
31
+ required: true,
32
+ });
33
+ }
34
+ for (const data of [
35
+ {
36
+ question: "Reply with exactly Paris and nothing else.",
37
+ expected: "Paris",
38
+ topic: "cities",
39
+ },
40
+ {
41
+ question: "Reply with exactly Tokyo and nothing else.",
42
+ expected: "Tokyo",
43
+ topic: "cities",
44
+ },
45
+ ]) {
46
+ await client.evaluations.datasets.cases.create(datasetId, {
47
+ datasetVersionId,
48
+ data,
49
+ tags: ["sdk-demo"],
50
+ });
51
+ }
52
+ await client.evaluations.datasets.versions.freeze(datasetId, datasetVersionId);
53
+ const suite = await client.evaluations.suites.create({
54
+ name: "SDK exact-match demo",
55
+ datasetVersionId,
56
+ });
57
+ suiteId = requireId(suite.id, "suite ID");
58
+ console.log({ suiteId });
59
+ const target = await client.evaluations.columns.create({
60
+ suiteId,
61
+ key: "candidate",
62
+ name: "Candidate",
63
+ category: "target",
64
+ enabled: true,
65
+ config: { role: "candidate", dataType: "output" },
66
+ target: {
67
+ type: "model",
68
+ name: "Demo model",
69
+ providerId: options.providerId,
70
+ modelId: options.modelId,
71
+ inputMapping: { user_message: "dataset.question" },
72
+ },
73
+ });
74
+ // The target's logical ID is distinct from the immutable version used in runs.
75
+ const targetId = requireId(target.target?.id, "target ID");
76
+ const targetVersionId = requireId(target.targetVersion?.id, "target version ID");
77
+ const evaluator = await client.evaluations.evaluators.create({
78
+ name: "SDK exact match",
79
+ version: {
80
+ type: "exact_match",
81
+ category: "deterministic",
82
+ scope: "output",
83
+ inputSchema: [
84
+ { key: "actual", type: "any", required: true },
85
+ { key: "expected", type: "any", required: true },
86
+ ],
87
+ config: {
88
+ caseSensitive: true,
89
+ trimWhitespace: true,
90
+ normalizeWhitespace: false,
91
+ },
92
+ outputType: "boolean",
93
+ scoreDirection: "none",
94
+ },
95
+ });
96
+ evaluatorId = requireId(evaluator.evaluator.id, "evaluator ID");
97
+ const evaluatorVersionId = requireId(evaluator.version?.id, "evaluator version ID");
98
+ console.log({ evaluatorId, targetVersionId, evaluatorVersionId });
99
+ await client.evaluations.columns.create({
100
+ suiteId,
101
+ key: "exact_match",
102
+ name: "Exact match",
103
+ category: "evaluator",
104
+ enabled: true,
105
+ evaluatorVersionId,
106
+ config: {
107
+ inputMapping: {
108
+ actual: { source: "target", path: "output" },
109
+ expected: { source: "dataset", key: "expected" },
110
+ },
111
+ },
112
+ });
113
+ const experiment = await client.evaluations.experiments.prepare({
114
+ suiteId,
115
+ datasetVersionId,
116
+ targetVersionIds: [targetVersionId],
117
+ evaluatorVersionIds: [evaluatorVersionId],
118
+ trialCount: 1,
119
+ });
120
+ const experimentId = requireId(experiment.id, "experiment ID");
121
+ // Persist this input and submissionId if implementing replay across process restarts.
122
+ const submissionId = crypto.randomUUID();
123
+ console.log({ experimentId, submissionId });
124
+ submissionStarted = true;
125
+ const created = await client.evaluations.runs.create({
126
+ experimentId,
127
+ cases: "all",
128
+ targetIds: [targetVersionId],
129
+ evaluatorIds: [evaluatorVersionId],
130
+ trialCount: 1,
131
+ }, { idempotencyKey: submissionId, maxRetries: 0 });
132
+ runId = created.id;
133
+ console.log({ runId });
134
+ for (let attempt = 0; attempt < (options.maxPolls ?? 60); attempt++) {
135
+ const run = await client.evaluations.runs.get(runId, {
136
+ timeoutMs: 10_000,
137
+ maxRetries: 0,
138
+ });
139
+ if (["COMPLETED", "FAILED", "CANCELLED"].includes(run.status)) {
140
+ terminal = true;
141
+ const results = await client.evaluations.runs.results(runId, {
142
+ page: 1,
143
+ pageSize: 25,
144
+ });
145
+ for (const trial of results.data) {
146
+ console.log({
147
+ caseId: trial.caseId,
148
+ trialStatus: trial.status,
149
+ error: trial.error,
150
+ });
151
+ for (const score of trial.scores) {
152
+ console.log({
153
+ evaluatorVersionId: score.evaluatorVersionId,
154
+ status: score.status,
155
+ value: score.value,
156
+ passed: score.passed,
157
+ error: score.error,
158
+ });
159
+ }
160
+ }
161
+ // Two cases × one target × one trial fit on one page.
162
+ if (run.status !== "COMPLETED")
163
+ throw new Error(`Run ${runId}: ${run.status}`);
164
+ if (results.data.length !== 2 ||
165
+ results.data.some((trial) => trial.status !== "COMPLETED" ||
166
+ trial.scores.length !== 1 ||
167
+ trial.scores.some((score) => score.status !== "COMPLETED" || score.error))) {
168
+ throw new Error("Run ended with missing or failed trials/scores; inspect results");
169
+ }
170
+ // Runtime GET returns persisted aggregates after terminal processing.
171
+ const summary = run.summary;
172
+ if (!summary || typeof summary.overallPassRate !== "number")
173
+ throw new Error("Missing aggregate scores; inspect the terminal run");
174
+ console.log({ overallPassRate: summary.overallPassRate, byEvaluator: summary.byEvaluator });
175
+ return { runId, results, summary }; // passed:false is a valid evaluation outcome.
176
+ }
177
+ await new Promise((resolve) => setTimeout(resolve, options.pollMs ?? 2000));
178
+ }
179
+ await client.evaluations.runs.cancel(runId);
180
+ throw new Error(`Polling limit reached; cancellation requested for ${runId}. Confirm terminal state before cleanup.`);
181
+ }
182
+ finally {
183
+ // Never archive inputs while background execution might still use them.
184
+ if (options.cleanup && (!submissionStarted || terminal)) {
185
+ const cleanup = await Promise.allSettled([
186
+ ...(suiteId ? [client.evaluations.suites.archive(suiteId)] : []),
187
+ ...(evaluatorId
188
+ ? [client.evaluations.evaluators.archive(evaluatorId)]
189
+ : []),
190
+ ...(datasetId ? [client.evaluations.datasets.archive(datasetId)] : []),
191
+ ]);
192
+ for (const result of cleanup)
193
+ if (result.status === "rejected")
194
+ console.error("Archive failed:", result.reason);
195
+ }
196
+ }
197
+ }
198
+ if (process.env.NIRMOS_RUN_EVALUATION_EXAMPLE === "1") {
199
+ const key = requireId(process.env.NIRMOS_API_KEY, "NIRMOS_API_KEY");
200
+ await evaluationQuickstart(new Nirmos({
201
+ auth: { type: "api-key", key },
202
+ apiBaseUrl: process.env.NIRMOS_API_URL,
203
+ runtimeBaseUrl: process.env.NIRMOS_RUNTIME_URL,
204
+ maxRetries: 0,
205
+ }), {
206
+ providerId: requireId(process.env.NIRMOS_PROVIDER_ID, "NIRMOS_PROVIDER_ID"),
207
+ modelId: requireId(process.env.NIRMOS_MODEL_ID, "NIRMOS_MODEL_ID"),
208
+ cleanup: process.env.EVALUATION_CLEANUP === "1",
209
+ });
210
+ }