@zenera/gym 1.1.40 → 1.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +169 -104
- package/dist/agents.d.ts +25 -0
- package/dist/agents.js +52 -0
- package/dist/bindings.d.ts +25 -0
- package/dist/bindings.js +81 -0
- package/dist/box.d.ts +39 -36
- package/dist/box.js +183 -217
- package/dist/cache.d.ts +24 -14
- package/dist/cache.js +102 -66
- package/dist/catalog.d.ts +83 -0
- package/dist/catalog.js +293 -0
- package/dist/command.d.ts +1 -0
- package/dist/command.js +235 -68
- package/dist/fake.d.ts +4 -0
- package/dist/fake.js +142 -0
- package/dist/generate.d.ts +28 -24
- package/dist/generate.js +417 -120
- package/dist/inputs.d.ts +12 -0
- package/dist/inputs.js +132 -0
- package/dist/knowledge.d.ts +50 -0
- package/dist/knowledge.js +199 -0
- package/dist/logger.d.ts +10 -8
- package/dist/logger.js +4 -3
- package/dist/paging.d.ts +2 -2
- package/dist/paging.js +2 -2
- package/dist/probe.d.ts +5 -5
- package/dist/probe.js +15 -15
- package/dist/prompt.d.ts +31 -18
- package/dist/prompt.js +160 -146
- package/dist/quiet.d.ts +2 -0
- package/dist/quiet.js +7 -0
- package/dist/router.js +5 -1
- package/dist/runtime.d.ts +160 -0
- package/dist/runtime.js +229 -0
- package/dist/schema.js +1 -2
- package/dist/server.js +98 -57
- package/dist/setup.d.ts +47 -6
- package/dist/setup.js +157 -21
- package/dist/spec.d.ts +4 -3
- package/dist/spec.js +73 -22
- package/dist/store.d.ts +78 -0
- package/dist/store.js +253 -0
- package/dist/synth.d.ts +42 -0
- package/dist/synth.js +589 -0
- package/dist/worker.d.ts +41 -0
- package/dist/worker.js +99 -0
- package/package.json +5 -3
- package/dist/envelope.d.ts +0 -13
- package/dist/envelope.js +0 -9
- package/dist/image.d.ts +0 -25
- package/dist/image.js +0 -81
package/dist/cache.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { Cache as Store } from '@zenera/cli/lib';
|
|
1
|
+
import { Cache as Store, paths } from '@zenera/cli/lib';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import { build, BuildFailed
|
|
3
|
+
import { build, BuildFailed } from "./generate.js";
|
|
4
4
|
import { saveBuildDump } from "./logger.js";
|
|
5
|
-
import {
|
|
5
|
+
import { CONTRACT_VERSION } from "./spec.js";
|
|
6
|
+
import { leaseOwner, WorldStore } from "./store.js";
|
|
6
7
|
// ---------------------------------------------------------------------------
|
|
7
8
|
// The cache
|
|
8
9
|
//
|
|
@@ -11,36 +12,45 @@ import { GENERATOR_RULES_VERSION } from "./spec.js";
|
|
|
11
12
|
// simply never asked for again — there is nothing to invalidate, which is the
|
|
12
13
|
// part of a cache that is usually wrong.
|
|
13
14
|
//
|
|
14
|
-
// The
|
|
15
|
-
// the
|
|
15
|
+
// The handler itself lives in the machine's shared cache rather than beside
|
|
16
|
+
// the box's workspace, so the same spec served from two directories is
|
|
16
17
|
// written once. The box root is scratch: a hit is copied back into it, because
|
|
17
|
-
// the
|
|
18
|
+
// the worker imports a file, not a string.
|
|
18
19
|
//
|
|
19
20
|
// The in-flight map is the other half and matters more than it looks: ten
|
|
20
21
|
// requests arriving together for an uncached operation must produce one build,
|
|
21
22
|
// not ten. A failed build is remembered too, for the same reason — an operation
|
|
22
23
|
// the model could not write for should not re-ask on every request.
|
|
23
24
|
// ---------------------------------------------------------------------------
|
|
24
|
-
export const GYM_KIND = 'gym-
|
|
25
|
+
export const GYM_KIND = 'gym-handler';
|
|
25
26
|
const DEFAULT_CONCURRENCY = 4;
|
|
27
|
+
/** Longer than any build; a crashed builder's lease lapses at once. */
|
|
28
|
+
const BUILD_LEASE_MS = 20 * 60_000;
|
|
29
|
+
const BUILD_POLL_MS = 500;
|
|
26
30
|
export class Cache {
|
|
27
31
|
#opts;
|
|
28
32
|
#store;
|
|
29
33
|
#dumpDir;
|
|
34
|
+
#reportDir;
|
|
30
35
|
#records = new Map();
|
|
31
36
|
#live = new Map();
|
|
32
37
|
#settled = new Set();
|
|
33
38
|
#slots;
|
|
39
|
+
#leaseFile;
|
|
40
|
+
#owner = leaseOwner();
|
|
41
|
+
#leaseStore;
|
|
34
42
|
#waiting = [];
|
|
35
43
|
#running = 0;
|
|
36
44
|
constructor(opts) {
|
|
37
45
|
this.#opts = opts;
|
|
38
46
|
this.#store = new Store(GYM_KIND, { dir: opts.cacheDir, mode: 0o600 });
|
|
47
|
+
this.#leaseFile = join(opts.cacheDir ?? paths.cache(), GYM_KIND, 'leases.sqlite');
|
|
39
48
|
this.#dumpDir = opts.dumpDir ?? join(opts.box.root, 'builds');
|
|
49
|
+
this.#reportDir = opts.reportDir ?? join(opts.box.root, 'reports');
|
|
40
50
|
this.#slots = Math.max(1, opts.concurrency ?? DEFAULT_CONCURRENCY);
|
|
41
51
|
}
|
|
42
52
|
/**
|
|
43
|
-
* The
|
|
53
|
+
* The handler for an operation, written if it does not exist yet. Callers
|
|
44
54
|
* that arrive during a build wait for that build rather than starting one.
|
|
45
55
|
*
|
|
46
56
|
* `cached` answers "did this call cost a model turn", which is what the
|
|
@@ -78,37 +88,12 @@ export class Cache {
|
|
|
78
88
|
return found?.source?.trim() ? found.source : undefined;
|
|
79
89
|
}
|
|
80
90
|
async regenerate(operation, ctx) {
|
|
81
|
-
const { box
|
|
91
|
+
const { box } = this.#opts;
|
|
82
92
|
await this.#enter();
|
|
83
93
|
this.#opts.onStart?.({ operation });
|
|
84
94
|
try {
|
|
85
|
-
const built = await
|
|
86
|
-
|
|
87
|
-
box,
|
|
88
|
-
checks,
|
|
89
|
-
attempts: this.#opts.attempts,
|
|
90
|
-
currentSource: ctx.currentSource,
|
|
91
|
-
failingInput: ctx.failingInput,
|
|
92
|
-
fault: ctx.fault,
|
|
93
|
-
stderr: ctx.stderr,
|
|
94
|
-
examples: ctx.examples,
|
|
95
|
-
onAttempt: (attempt, diagnostics, source, limit) => this.#rejected(operation, attempt, diagnostics, source, limit, true),
|
|
96
|
-
});
|
|
97
|
-
if (!ephemeral) {
|
|
98
|
-
this.#store.put(operation.key, {
|
|
99
|
-
source: built.source,
|
|
100
|
-
meta: {
|
|
101
|
-
operationId: operation.operationId,
|
|
102
|
-
method: operation.method,
|
|
103
|
-
path: operation.path,
|
|
104
|
-
source: operation.source,
|
|
105
|
-
model: model.id,
|
|
106
|
-
attempts: built.attempts,
|
|
107
|
-
rulesVersion: GENERATOR_RULES_VERSION,
|
|
108
|
-
createdAt: new Date().toISOString(),
|
|
109
|
-
},
|
|
110
|
-
});
|
|
111
|
-
}
|
|
95
|
+
const built = await build(operation, this.#builder(operation, true), ctx);
|
|
96
|
+
this.#keep(operation, built);
|
|
112
97
|
this.#opts.onReady?.({ operation, cached: false, attempts: built.attempts });
|
|
113
98
|
const gen = { key: operation.key, source: built.source, cached: false };
|
|
114
99
|
this.#settled.add(operation.key);
|
|
@@ -128,8 +113,44 @@ export class Cache {
|
|
|
128
113
|
this.#leave();
|
|
129
114
|
}
|
|
130
115
|
}
|
|
116
|
+
#builder(operation, regeneration) {
|
|
117
|
+
const { box, model, checks, catalog, world, operations, bindings } = this.#opts;
|
|
118
|
+
return {
|
|
119
|
+
model,
|
|
120
|
+
box,
|
|
121
|
+
checks,
|
|
122
|
+
catalog,
|
|
123
|
+
world,
|
|
124
|
+
bindings,
|
|
125
|
+
operations,
|
|
126
|
+
attempts: this.#opts.attempts,
|
|
127
|
+
reportDir: this.#reportDir,
|
|
128
|
+
sourceOf: (other) => this.getSource(other.key),
|
|
129
|
+
onAttempt: (attempt, diagnostics, source, limit) => this.#rejected(operation, attempt, diagnostics, source, limit, regeneration),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
#keep(operation, built) {
|
|
133
|
+
if (this.#opts.ephemeral) {
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
const { source, attempts, bindings } = built;
|
|
137
|
+
this.#store.put(operation.key, {
|
|
138
|
+
source,
|
|
139
|
+
bindings,
|
|
140
|
+
meta: {
|
|
141
|
+
operationId: operation.operationId,
|
|
142
|
+
method: operation.method,
|
|
143
|
+
path: operation.path,
|
|
144
|
+
source: operation.source,
|
|
145
|
+
model: this.#opts.model.id,
|
|
146
|
+
attempts,
|
|
147
|
+
rulesVersion: CONTRACT_VERSION,
|
|
148
|
+
createdAt: new Date().toISOString(),
|
|
149
|
+
},
|
|
150
|
+
});
|
|
151
|
+
}
|
|
131
152
|
async #make(operation) {
|
|
132
|
-
const { box,
|
|
153
|
+
const { box, rebuild } = this.#opts;
|
|
133
154
|
// Read before queueing: a cache hit costs nothing and must not wait
|
|
134
155
|
// behind somebody else's model call.
|
|
135
156
|
const source = rebuild ? undefined : await this.#read(box, operation.key);
|
|
@@ -137,31 +158,30 @@ export class Cache {
|
|
|
137
158
|
this.#opts.onReady?.({ operation, cached: true, attempts: 0 });
|
|
138
159
|
return { key: operation.key, source, cached: true };
|
|
139
160
|
}
|
|
161
|
+
// Another process writing the same handler — same documents, other
|
|
162
|
+
// port — is waited for, and its handler read, rather than paid for twice.
|
|
163
|
+
const lease = `build:${operation.key}`;
|
|
164
|
+
while (!this.#leases().claim(lease, this.#owner, BUILD_LEASE_MS)) {
|
|
165
|
+
await new Promise((settle) => setTimeout(settle, BUILD_POLL_MS));
|
|
166
|
+
const theirs = await this.#read(box, operation.key);
|
|
167
|
+
if (theirs !== undefined) {
|
|
168
|
+
this.#opts.onReady?.({ operation, cached: true, attempts: 0 });
|
|
169
|
+
return { key: operation.key, source: theirs, cached: true };
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
return await this.#write(operation);
|
|
174
|
+
}
|
|
175
|
+
finally {
|
|
176
|
+
this.#leases().release(lease, this.#owner);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
async #write(operation) {
|
|
140
180
|
await this.#enter();
|
|
141
181
|
this.#opts.onStart?.({ operation });
|
|
142
182
|
try {
|
|
143
|
-
const built = await build(operation,
|
|
144
|
-
|
|
145
|
-
box,
|
|
146
|
-
checks,
|
|
147
|
-
attempts: this.#opts.attempts,
|
|
148
|
-
onAttempt: (attempt, diagnostics, source, limit) => this.#rejected(operation, attempt, diagnostics, source, limit, false),
|
|
149
|
-
});
|
|
150
|
-
if (!ephemeral) {
|
|
151
|
-
this.#store.put(operation.key, {
|
|
152
|
-
source: built.source,
|
|
153
|
-
meta: {
|
|
154
|
-
operationId: operation.operationId,
|
|
155
|
-
method: operation.method,
|
|
156
|
-
path: operation.path,
|
|
157
|
-
source: operation.source,
|
|
158
|
-
model: model.id,
|
|
159
|
-
attempts: built.attempts,
|
|
160
|
-
rulesVersion: GENERATOR_RULES_VERSION,
|
|
161
|
-
createdAt: new Date().toISOString(),
|
|
162
|
-
},
|
|
163
|
-
});
|
|
164
|
-
}
|
|
183
|
+
const built = await build(operation, this.#builder(operation, false));
|
|
184
|
+
this.#keep(operation, built);
|
|
165
185
|
this.#opts.onReady?.({ operation, cached: false, attempts: built.attempts });
|
|
166
186
|
await this.#finished(operation);
|
|
167
187
|
return { key: operation.key, source: built.source, cached: false };
|
|
@@ -178,9 +198,9 @@ export class Cache {
|
|
|
178
198
|
}
|
|
179
199
|
/**
|
|
180
200
|
* An attempt the judge threw out. The report is rewritten with the file
|
|
181
|
-
* still in it before the next
|
|
182
|
-
* that source is the only evidence there will ever be, since a
|
|
183
|
-
* that hangs leaves no
|
|
201
|
+
* still in it before the next test overwrites `handler.ts` — for a timeout
|
|
202
|
+
* that source is the only evidence there will ever be, since a handler
|
|
203
|
+
* that hangs leaves no stack behind.
|
|
184
204
|
*/
|
|
185
205
|
async #rejected(operation, attempt, diagnostics, source, limit, regeneration) {
|
|
186
206
|
let record = this.#records.get(operation.key);
|
|
@@ -199,7 +219,7 @@ export class Cache {
|
|
|
199
219
|
this.#opts.onAttempt?.({ operation, attempt, diagnostics, dump: record.path });
|
|
200
220
|
}
|
|
201
221
|
/** A report that cannot be written is not worth failing a build over. */
|
|
202
|
-
async #report(record, gaveUp = false) {
|
|
222
|
+
async #report(record, gaveUp = false, report) {
|
|
203
223
|
try {
|
|
204
224
|
record.path = await saveBuildDump(this.#dumpDir, {
|
|
205
225
|
started: record.started,
|
|
@@ -212,6 +232,7 @@ export class Cache {
|
|
|
212
232
|
model: this.#opts.model.id,
|
|
213
233
|
attempts: record.attempts,
|
|
214
234
|
gaveUp,
|
|
235
|
+
report,
|
|
215
236
|
});
|
|
216
237
|
}
|
|
217
238
|
catch {
|
|
@@ -230,14 +251,14 @@ export class Cache {
|
|
|
230
251
|
return undefined;
|
|
231
252
|
}
|
|
232
253
|
if (error instanceof BuildFailed) {
|
|
233
|
-
await this.#report(record, true);
|
|
254
|
+
await this.#report(record, true, error.report);
|
|
234
255
|
error.dump = record.path;
|
|
235
256
|
}
|
|
236
257
|
return record.path;
|
|
237
258
|
}
|
|
238
259
|
/**
|
|
239
|
-
* A hit is written into the box before it is returned. The
|
|
240
|
-
* files
|
|
260
|
+
* A hit is written into the box before it is returned. The worker imports
|
|
261
|
+
* files from the box and nothing else, and the box is scratch that any
|
|
241
262
|
* `cache clear` is free to delete.
|
|
242
263
|
*/
|
|
243
264
|
async #read(box, key) {
|
|
@@ -245,6 +266,16 @@ export class Cache {
|
|
|
245
266
|
if (!found?.source?.trim()) {
|
|
246
267
|
return undefined;
|
|
247
268
|
}
|
|
269
|
+
// Written against a placement that no longer holds: it would read and
|
|
270
|
+
// write somewhere nobody else looks.
|
|
271
|
+
const bindings = Object.entries(found.bindings ?? {});
|
|
272
|
+
if (bindings.some(([type, b]) => this.#opts.bindings.conflict(type, b))) {
|
|
273
|
+
return undefined;
|
|
274
|
+
}
|
|
275
|
+
// A cleared bindings store gets back what its handlers relied on.
|
|
276
|
+
for (const [type, b] of bindings) {
|
|
277
|
+
this.#opts.bindings.bind(type, b);
|
|
278
|
+
}
|
|
248
279
|
try {
|
|
249
280
|
await box.write(key, found.source);
|
|
250
281
|
}
|
|
@@ -254,6 +285,11 @@ export class Cache {
|
|
|
254
285
|
}
|
|
255
286
|
return found.source;
|
|
256
287
|
}
|
|
288
|
+
/** Opened on the first build, so a server that never writes one never opens it. */
|
|
289
|
+
#leases() {
|
|
290
|
+
this.#leaseStore ??= new WorldStore(this.#leaseFile);
|
|
291
|
+
return this.#leaseStore;
|
|
292
|
+
}
|
|
257
293
|
#enter() {
|
|
258
294
|
if (this.#running < this.#slots) {
|
|
259
295
|
this.#running++;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { AnyTool } from '@zenera/neo';
|
|
2
|
+
import { type ValidateFunction } from 'ajv/dist/2020.js';
|
|
3
|
+
import * as rag from '@zenera/rag';
|
|
4
|
+
import type { Operation } from './spec.ts';
|
|
5
|
+
export declare class Catalog {
|
|
6
|
+
#private;
|
|
7
|
+
readonly corpus: rag.Corpus;
|
|
8
|
+
readonly graph: rag.ApiGraph;
|
|
9
|
+
readonly types: Record<string, rag.Schema>;
|
|
10
|
+
private constructor();
|
|
11
|
+
static open(files: readonly string[]): Promise<Catalog>;
|
|
12
|
+
hasType(name: string): boolean;
|
|
13
|
+
/** A validator for records of a named type, or undefined for a name nobody declared. */
|
|
14
|
+
validator(name: string): ValidateFunction | undefined;
|
|
15
|
+
/** A validator for a schema of gym's own, outside the documents. */
|
|
16
|
+
compile(schema: object): ValidateFunction;
|
|
17
|
+
/** The named types and everything they refer to, as TypeScript. */
|
|
18
|
+
declare(names: readonly string[], maxChars?: number): {
|
|
19
|
+
code: string;
|
|
20
|
+
omitted: string[];
|
|
21
|
+
};
|
|
22
|
+
/** Names reachable from these through `$ref`, the named ones first. */
|
|
23
|
+
closure(names: readonly string[]): string[];
|
|
24
|
+
/** An inline type expression for a schema, e.g. `Message[]`. */
|
|
25
|
+
signature(schema: unknown): string;
|
|
26
|
+
/** The document's own record of an operation, matched by method and path. */
|
|
27
|
+
find(operation: Operation): rag.Operation | undefined;
|
|
28
|
+
/**
|
|
29
|
+
* The operation's request and response as TypeScript: what an agent writing
|
|
30
|
+
* its handler needs to read first, and usually all it needs.
|
|
31
|
+
*/
|
|
32
|
+
describe(operation: Operation): string;
|
|
33
|
+
/**
|
|
34
|
+
* `list_api`, `grep_api`, `trace_api` and `find_types_with_property` over
|
|
35
|
+
* the graph — exact, and needing no embedder. `describe_types` is gym's own.
|
|
36
|
+
*/
|
|
37
|
+
tools(): AnyTool[];
|
|
38
|
+
/**
|
|
39
|
+
* The type of the records an operation is about: its response type, or —
|
|
40
|
+
* when that is a list envelope with no identity of its own — the type its
|
|
41
|
+
* list holds. `Tier0ListResult` is about `Tier0`; `Event` is about `Event`.
|
|
42
|
+
*/
|
|
43
|
+
itemType(operation: Operation): string | undefined;
|
|
44
|
+
/**
|
|
45
|
+
* A type's own properties and the ones it inherits through `allOf`, which is
|
|
46
|
+
* how most documents spell inheritance — an NSX `Tier0` declares little of
|
|
47
|
+
* what it holds itself.
|
|
48
|
+
*/
|
|
49
|
+
properties(type: string): {
|
|
50
|
+
name: string;
|
|
51
|
+
schema: rag.Schema;
|
|
52
|
+
required: boolean;
|
|
53
|
+
}[];
|
|
54
|
+
/** The property a type's records are known by. */
|
|
55
|
+
idField(type: string): string;
|
|
56
|
+
/**
|
|
57
|
+
* Fields that point at other entities, judged by name: `*_id`, `*_path`,
|
|
58
|
+
* `*Ref`, `parent`, `owner`… Names rather than the schema because the
|
|
59
|
+
* schema rarely says: an NSX `connectivity_path` is a string, and only its
|
|
60
|
+
* name and its description say it names a Tier-1.
|
|
61
|
+
*/
|
|
62
|
+
references(type: string): {
|
|
63
|
+
name: string;
|
|
64
|
+
signature: string;
|
|
65
|
+
doc: string;
|
|
66
|
+
}[];
|
|
67
|
+
/**
|
|
68
|
+
* The type a reference field most likely names, from the field's name:
|
|
69
|
+
* `labelIds` → `Label`, `transport_zone_path` → `PolicyTransportZone`. A
|
|
70
|
+
* field that names no type — a `threadId` where threads are not a schema —
|
|
71
|
+
* has no referent, and cannot dangle.
|
|
72
|
+
*/
|
|
73
|
+
referent(field: string): string | undefined;
|
|
74
|
+
/** Every operation that accepts or returns a type, as method and path. */
|
|
75
|
+
carriers(type: string, maxHops?: number): {
|
|
76
|
+
method: string;
|
|
77
|
+
path: string;
|
|
78
|
+
direction: string;
|
|
79
|
+
}[];
|
|
80
|
+
/** The component types a response carries, nearest first: the candidates for a collection. */
|
|
81
|
+
responseTypes(operation: Operation): string[];
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=catalog.d.ts.map
|
package/dist/catalog.js
ADDED
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import addFormatsDefault from 'ajv-formats';
|
|
2
|
+
import { Ajv2020 } from 'ajv/dist/2020.js';
|
|
3
|
+
import * as rag from '@zenera/rag';
|
|
4
|
+
const addFormats = addFormatsDefault;
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
// The catalog
|
|
7
|
+
//
|
|
8
|
+
// The documents as `@zenera/rag` reads them: bundled rather than dereferenced,
|
|
9
|
+
// so `Message` stays one named type instead of being pasted into every
|
|
10
|
+
// operation that returns it. That is what lets an agent be shown a response as
|
|
11
|
+
// a few TypeScript interfaces instead of a page of JSON Schema, ask for a
|
|
12
|
+
// nested type by name when it needs one, and be told that the records it makes
|
|
13
|
+
// for a collection must be a `Message` — validated here against that one name.
|
|
14
|
+
//
|
|
15
|
+
// Nothing here is embedded or indexed. Reading and graphing a document is
|
|
16
|
+
// milliseconds even at thousands of operations; only search needs vectors, and
|
|
17
|
+
// nothing here searches.
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
/** A brief stops quoting declarations past this, and points at describe_types. */
|
|
20
|
+
const BRIEF_CHARS = 12_000;
|
|
21
|
+
const TYPES_ID = 'gym:types';
|
|
22
|
+
export class Catalog {
|
|
23
|
+
corpus;
|
|
24
|
+
graph;
|
|
25
|
+
types;
|
|
26
|
+
#printer;
|
|
27
|
+
#ajv;
|
|
28
|
+
#validators = new Map();
|
|
29
|
+
constructor(corpus) {
|
|
30
|
+
this.corpus = corpus;
|
|
31
|
+
const built = rag.buildGraph(corpus);
|
|
32
|
+
this.graph = built.graph;
|
|
33
|
+
this.types = built.types;
|
|
34
|
+
this.#printer = new rag.Printer(this.types);
|
|
35
|
+
this.#ajv = new Ajv2020({ allErrors: true, strict: false, logger: false });
|
|
36
|
+
addFormats(this.#ajv);
|
|
37
|
+
this.#ajv.addSchema({ $id: TYPES_ID, $defs: stripSchemaKeys(this.types) });
|
|
38
|
+
}
|
|
39
|
+
static async open(files) {
|
|
40
|
+
return new Catalog(await rag.loadSpecs(files));
|
|
41
|
+
}
|
|
42
|
+
hasType(name) {
|
|
43
|
+
return name in this.types;
|
|
44
|
+
}
|
|
45
|
+
/** A validator for records of a named type, or undefined for a name nobody declared. */
|
|
46
|
+
validator(name) {
|
|
47
|
+
if (!this.hasType(name)) {
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
let found = this.#validators.get(name);
|
|
51
|
+
if (!found) {
|
|
52
|
+
found = this.#ajv.compile({ $ref: `${TYPES_ID}#/$defs/${pointer(name)}` });
|
|
53
|
+
this.#validators.set(name, found);
|
|
54
|
+
}
|
|
55
|
+
return found;
|
|
56
|
+
}
|
|
57
|
+
/** A validator for a schema of gym's own, outside the documents. */
|
|
58
|
+
compile(schema) {
|
|
59
|
+
return this.#ajv.compile(schema);
|
|
60
|
+
}
|
|
61
|
+
/** The named types and everything they refer to, as TypeScript. */
|
|
62
|
+
declare(names, maxChars = Infinity) {
|
|
63
|
+
const wanted = this.closure(names);
|
|
64
|
+
const blocks = [];
|
|
65
|
+
const omitted = [];
|
|
66
|
+
let size = 0;
|
|
67
|
+
for (const name of wanted) {
|
|
68
|
+
const block = this.#printer.declaration(name, { docs: true, maxDoc: 160 });
|
|
69
|
+
if (size + block.length > maxChars && blocks.length > 0) {
|
|
70
|
+
omitted.push(name);
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
blocks.push(block);
|
|
74
|
+
size += block.length;
|
|
75
|
+
}
|
|
76
|
+
return { code: blocks.join('\n'), omitted };
|
|
77
|
+
}
|
|
78
|
+
/** Names reachable from these through `$ref`, the named ones first. */
|
|
79
|
+
closure(names) {
|
|
80
|
+
const out = [];
|
|
81
|
+
const seen = new Set();
|
|
82
|
+
const queue = names.filter((n) => this.hasType(n));
|
|
83
|
+
while (queue.length > 0) {
|
|
84
|
+
const name = queue.shift();
|
|
85
|
+
if (seen.has(name)) {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
seen.add(name);
|
|
89
|
+
out.push(name);
|
|
90
|
+
for (const ref of rag.refsIn(this.types[name])) {
|
|
91
|
+
if (this.hasType(ref) && !seen.has(ref)) {
|
|
92
|
+
queue.push(ref);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
/** An inline type expression for a schema, e.g. `Message[]`. */
|
|
99
|
+
signature(schema) {
|
|
100
|
+
return this.#printer.signature(schema);
|
|
101
|
+
}
|
|
102
|
+
/** The document's own record of an operation, matched by method and path. */
|
|
103
|
+
find(operation) {
|
|
104
|
+
const candidates = this.corpus.operations.filter((o) => o.method === operation.method && o.path.split('?')[0] === operation.path);
|
|
105
|
+
return (candidates.find((o) => o.operationId === operation.operationId) ??
|
|
106
|
+
candidates.find((o) => o.path.includes('?') === Boolean(operation.fixed)) ??
|
|
107
|
+
candidates[0]);
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* The operation's request and response as TypeScript: what an agent writing
|
|
111
|
+
* its handler needs to read first, and usually all it needs.
|
|
112
|
+
*/
|
|
113
|
+
describe(operation) {
|
|
114
|
+
const op = this.find(operation);
|
|
115
|
+
if (!op) {
|
|
116
|
+
return '';
|
|
117
|
+
}
|
|
118
|
+
const lines = [];
|
|
119
|
+
const roots = [];
|
|
120
|
+
const note = (schema) => {
|
|
121
|
+
roots.push(...rag.refsIn(schema));
|
|
122
|
+
return this.signature(schema);
|
|
123
|
+
};
|
|
124
|
+
for (const p of op.params) {
|
|
125
|
+
lines.push(` ${p.in} ${p.name}${p.required ? '' : '?'}: ${note(p.schema)}${p.doc ? ` // ${oneLine(p.doc)}` : ''}`);
|
|
126
|
+
}
|
|
127
|
+
if (op.requestBody) {
|
|
128
|
+
lines.push(` body${op.requestBody.required ? '' : '?'}: ${note(op.requestBody.schema)}`);
|
|
129
|
+
}
|
|
130
|
+
const response = op.responses[0];
|
|
131
|
+
const out = [
|
|
132
|
+
'REQUEST',
|
|
133
|
+
...(lines.length > 0 ? lines : [' (no parameters)']),
|
|
134
|
+
'',
|
|
135
|
+
`RESPONSE ${response ? `${response.status}: ${note(response.schema)}` : '(no body)'}`,
|
|
136
|
+
];
|
|
137
|
+
const declared = this.declare([...new Set(roots)], BRIEF_CHARS);
|
|
138
|
+
if (declared.code) {
|
|
139
|
+
out.push('', 'TYPES', declared.code.trimEnd());
|
|
140
|
+
}
|
|
141
|
+
if (declared.omitted.length > 0) {
|
|
142
|
+
out.push('', `Not shown for length (ask describe_types): ${declared.omitted.join(', ')}`);
|
|
143
|
+
}
|
|
144
|
+
return out.join('\n');
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* `list_api`, `grep_api`, `trace_api` and `find_types_with_property` over
|
|
148
|
+
* the graph — exact, and needing no embedder. `describe_types` is gym's own.
|
|
149
|
+
*/
|
|
150
|
+
tools() {
|
|
151
|
+
return rag
|
|
152
|
+
.graphTools({
|
|
153
|
+
graph: this.graph,
|
|
154
|
+
sources: this.corpus.docs.map((d) => d.source),
|
|
155
|
+
schemas: () => Promise.resolve(this.types),
|
|
156
|
+
})
|
|
157
|
+
.filter((t) => t.name !== 'describe_types');
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* The type of the records an operation is about: its response type, or —
|
|
161
|
+
* when that is a list envelope with no identity of its own — the type its
|
|
162
|
+
* list holds. `Tier0ListResult` is about `Tier0`; `Event` is about `Event`.
|
|
163
|
+
*/
|
|
164
|
+
itemType(operation) {
|
|
165
|
+
const op = this.find(operation);
|
|
166
|
+
const schema = op?.responses[0]?.schema ?? op?.requestBody?.schema;
|
|
167
|
+
const root = rag.refName(schema) ?? rag.refName(isObject(schema) ? schema.items : undefined);
|
|
168
|
+
const type = root ? this.types[root] : undefined;
|
|
169
|
+
if (!root || !type) {
|
|
170
|
+
return undefined;
|
|
171
|
+
}
|
|
172
|
+
const properties = isObject(type.properties) ? type.properties : {};
|
|
173
|
+
if (Object.keys(properties).some((k) => IDENTITY.test(k))) {
|
|
174
|
+
return root;
|
|
175
|
+
}
|
|
176
|
+
const lists = Object.entries(properties)
|
|
177
|
+
.map(([name, p]) => [name, isObject(p) ? rag.refName(p.items) : undefined])
|
|
178
|
+
.filter((entry) => Boolean(entry[1]));
|
|
179
|
+
const preferred = lists.find(([name]) => LIST_NAMES.test(name)) ?? lists[0];
|
|
180
|
+
return preferred?.[1] ?? root;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* A type's own properties and the ones it inherits through `allOf`, which is
|
|
184
|
+
* how most documents spell inheritance — an NSX `Tier0` declares little of
|
|
185
|
+
* what it holds itself.
|
|
186
|
+
*/
|
|
187
|
+
properties(type) {
|
|
188
|
+
const out = new Map();
|
|
189
|
+
const visit = (schema, depth) => {
|
|
190
|
+
if (!isObject(schema) || depth > 6) {
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
const ref = rag.refName(schema);
|
|
194
|
+
if (ref) {
|
|
195
|
+
visit(this.types[ref], depth + 1);
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
for (const member of Array.isArray(schema.allOf) ? schema.allOf : []) {
|
|
199
|
+
visit(member, depth + 1);
|
|
200
|
+
}
|
|
201
|
+
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
|
|
202
|
+
for (const [name, sub] of Object.entries(isObject(schema.properties) ? schema.properties : {})) {
|
|
203
|
+
if (isObject(sub)) {
|
|
204
|
+
out.set(name, { name, schema: sub, required: required.has(name) });
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
visit(this.types[type], 0);
|
|
209
|
+
return [...out.values()];
|
|
210
|
+
}
|
|
211
|
+
/** The property a type's records are known by. */
|
|
212
|
+
idField(type) {
|
|
213
|
+
const names = this.properties(type).map((p) => p.name);
|
|
214
|
+
return names.find((n) => n === 'id') ?? names.find((n) => IDENTITY.test(n)) ?? 'id';
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Fields that point at other entities, judged by name: `*_id`, `*_path`,
|
|
218
|
+
* `*Ref`, `parent`, `owner`… Names rather than the schema because the
|
|
219
|
+
* schema rarely says: an NSX `connectivity_path` is a string, and only its
|
|
220
|
+
* name and its description say it names a Tier-1.
|
|
221
|
+
*/
|
|
222
|
+
references(type) {
|
|
223
|
+
const own = this.idField(type);
|
|
224
|
+
return this.properties(type)
|
|
225
|
+
.filter((p) => p.name !== own && REFERENCE.test(p.name) && stringy(p.schema))
|
|
226
|
+
.map((p) => ({
|
|
227
|
+
name: p.name,
|
|
228
|
+
signature: this.signature(p.schema),
|
|
229
|
+
doc: oneLine(typeof p.schema.description === 'string' ? p.schema.description : ''),
|
|
230
|
+
}));
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* The type a reference field most likely names, from the field's name:
|
|
234
|
+
* `labelIds` → `Label`, `transport_zone_path` → `PolicyTransportZone`. A
|
|
235
|
+
* field that names no type — a `threadId` where threads are not a schema —
|
|
236
|
+
* has no referent, and cannot dangle.
|
|
237
|
+
*/
|
|
238
|
+
referent(field) {
|
|
239
|
+
const base = squash(field.replace(REFERENCE, ''));
|
|
240
|
+
if (!base) {
|
|
241
|
+
return undefined;
|
|
242
|
+
}
|
|
243
|
+
const types = Object.keys(this.types);
|
|
244
|
+
return (types.find((t) => squash(t) === base) ??
|
|
245
|
+
types.filter((t) => squash(t).endsWith(base)).sort((a, b) => a.length - b.length)[0]);
|
|
246
|
+
}
|
|
247
|
+
/** Every operation that accepts or returns a type, as method and path. */
|
|
248
|
+
carriers(type, maxHops = 4) {
|
|
249
|
+
const id = `Type:${type}`;
|
|
250
|
+
if (!this.graph.hasNode(id)) {
|
|
251
|
+
return [];
|
|
252
|
+
}
|
|
253
|
+
const traced = rag.traceNodes(this.graph, { ids: [id], maxHops, maxRoutes: 100 });
|
|
254
|
+
return (traced.traces[0]?.routes ?? []).map((r) => ({
|
|
255
|
+
method: r.attributes.httpMethod.toLowerCase(),
|
|
256
|
+
path: r.attributes.path.split('?')[0],
|
|
257
|
+
direction: r.direction,
|
|
258
|
+
}));
|
|
259
|
+
}
|
|
260
|
+
/** The component types a response carries, nearest first: the candidates for a collection. */
|
|
261
|
+
responseTypes(operation) {
|
|
262
|
+
const op = this.find(operation);
|
|
263
|
+
const schema = op?.responses[0]?.schema;
|
|
264
|
+
return schema ? this.closure(rag.refsIn(schema)) : [];
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
/** A subschema must not carry its own `$schema`; only the root may. */
|
|
268
|
+
function stripSchemaKeys(types) {
|
|
269
|
+
return Object.fromEntries(Object.entries(types).map(([name, schema]) => {
|
|
270
|
+
const { $schema: _a, $id: _b, ...rest } = schema;
|
|
271
|
+
return [name, rest];
|
|
272
|
+
}));
|
|
273
|
+
}
|
|
274
|
+
/** A property that makes an object a record rather than an envelope. */
|
|
275
|
+
const IDENTITY = /^(id|uid|uuid|_id|key|name)$/i;
|
|
276
|
+
/** A field named for the thing it points at. */
|
|
277
|
+
const REFERENCE = /(_id|Id|_ids|Ids|_path|Path|_paths|Paths|_ref|Ref|_refs|Refs|_uri|Uri)$|^(parent|owner|organizer|assignee|author|creator|manager)$/;
|
|
278
|
+
const squash = (name) => name.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
279
|
+
/** A string, or a list of them: what an id or a path is written as. */
|
|
280
|
+
function stringy(schema) {
|
|
281
|
+
const type = schema.type;
|
|
282
|
+
if (type === 'string' || (Array.isArray(type) && type.includes('string'))) {
|
|
283
|
+
return true;
|
|
284
|
+
}
|
|
285
|
+
const items = schema.items;
|
|
286
|
+
return type === 'array' && isObject(items) && items.type === 'string';
|
|
287
|
+
}
|
|
288
|
+
/** What an envelope usually calls its list. */
|
|
289
|
+
const LIST_NAMES = /^(items|results|data|records|entries|values|objects|list|content)$/i;
|
|
290
|
+
const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
291
|
+
const pointer = (name) => encodeURIComponent(name.replace(/~/g, '~0').replace(/\//g, '~1'));
|
|
292
|
+
const oneLine = (text) => text.replace(/\s+/g, ' ').trim().slice(0, 120);
|
|
293
|
+
//# sourceMappingURL=catalog.js.map
|
package/dist/command.d.ts
CHANGED