@volter/twin-standard 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +20 -0
- package/dist/src/check-sources.d.ts +33 -0
- package/dist/src/check-sources.js +128 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +105 -0
- package/dist/src/derive.d.ts +2 -0
- package/dist/src/derive.js +349 -0
- package/dist/src/gate.d.ts +9 -0
- package/dist/src/gate.js +109 -0
- package/dist/src/grade.d.ts +30 -0
- package/dist/src/grade.js +36 -0
- package/dist/src/index.d.ts +7 -0
- package/dist/src/index.js +11 -0
- package/dist/src/lanes.d.ts +2 -0
- package/dist/src/lanes.js +33 -0
- package/dist/src/p3-rules.d.ts +8 -0
- package/dist/src/p3-rules.js +43 -0
- package/dist/src/protocol-3.d.ts +5 -0
- package/dist/src/protocol-3.js +721 -0
- package/dist/src/published.d.ts +13 -0
- package/dist/src/published.js +36 -0
- package/dist/src/spec-documents.d.ts +12 -0
- package/dist/src/spec-documents.js +61 -0
- package/dist/src/spec-ir-client.d.ts +20 -0
- package/dist/src/spec-ir-client.js +77 -0
- package/dist/src/spec-ir-commands.d.ts +38 -0
- package/dist/src/spec-ir-commands.js +38 -0
- package/dist/src/spec-ir-discovery.d.ts +11 -0
- package/dist/src/spec-ir-discovery.js +106 -0
- package/dist/src/spec-ir-graphql.d.ts +25 -0
- package/dist/src/spec-ir-graphql.js +46 -0
- package/dist/src/spec-ir-lines.d.ts +19 -0
- package/dist/src/spec-ir-lines.js +76 -0
- package/dist/src/spec-ir-proto.d.ts +80 -0
- package/dist/src/spec-ir-proto.js +339 -0
- package/dist/src/spec-ir.d.ts +104 -0
- package/dist/src/spec-ir.js +691 -0
- package/dist/src/spec-patches.d.ts +8 -0
- package/dist/src/spec-patches.js +24 -0
- package/dist/src/spec.d.ts +5 -0
- package/dist/src/spec.js +7 -0
- package/dist/src/types.d.ts +75 -0
- package/dist/src/types.js +4 -0
- package/dist/src/unit.d.ts +5 -0
- package/dist/src/unit.js +14 -0
- package/package.json +71 -0
- package/src/check-sources.ts +109 -0
- package/src/cli.ts +75 -0
- package/src/derive.ts +316 -0
- package/src/gate.ts +95 -0
- package/src/grade.ts +47 -0
- package/src/index.ts +12 -0
- package/src/lanes.ts +29 -0
- package/src/p3-rules.ts +44 -0
- package/src/protocol-3.ts +617 -0
- package/src/published.ts +37 -0
- package/src/spec-documents.ts +58 -0
- package/src/spec-ir-client.ts +86 -0
- package/src/spec-ir-commands.ts +50 -0
- package/src/spec-ir-discovery.ts +104 -0
- package/src/spec-ir-graphql.ts +64 -0
- package/src/spec-ir-lines.ts +76 -0
- package/src/spec-ir-proto.ts +289 -0
- package/src/spec-ir.ts +689 -0
- package/src/spec-patches.ts +23 -0
- package/src/spec.ts +8 -0
- package/src/types.ts +53 -0
- package/src/unit.ts +15 -0
package/src/published.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// A published version graded as the catalog grades a submission (docs/contributing/architecture.md, "The catalog: where
|
|
2
|
+
// twins come from"): its tarball fetched from the registry, unpacked under its vendor's name (so its lanes are read as
|
|
3
|
+
// the vendor's), and every unit in it — the pack and each lane — put through the gate.
|
|
4
|
+
import { spawnSync } from 'node:child_process';
|
|
5
|
+
import { existsSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync } from 'node:fs';
|
|
6
|
+
import { tmpdir } from 'node:os';
|
|
7
|
+
import { join } from 'node:path';
|
|
8
|
+
import { p3Gate } from './gate.ts';
|
|
9
|
+
|
|
10
|
+
export type PublishedGate = { spec: string; failures: Array<{ unit: string; failures: string[] }> };
|
|
11
|
+
|
|
12
|
+
/** The vendor a package twins: its name without its scope and the `twin-` prefix (`@volter/twin-supabase` is supabase). */
|
|
13
|
+
export const vendorOfPackage = (name: string): string => name.replace(/^@[^/]+\//, '').replace(/^twin-/, '');
|
|
14
|
+
|
|
15
|
+
/** The gate over `spec` (`<name>@<version>`) as the registry serves it; throws when it cannot be fetched or unpacked. */
|
|
16
|
+
export async function gatePublished(spec: string, opts: { registry?: string } = {}): Promise<PublishedGate> {
|
|
17
|
+
const tmp = mkdtempSync(join(tmpdir(), 'twin-standard-'));
|
|
18
|
+
try {
|
|
19
|
+
const pack = spawnSync('npm', ['pack', spec, '--pack-destination', tmp, '--silent', ...(opts.registry ? ['--registry', opts.registry] : [])], { encoding: 'utf8' });
|
|
20
|
+
const tarball = readdirSync(tmp).find((f) => f.endsWith('.tgz'));
|
|
21
|
+
if (pack.status !== 0 || !tarball) throw new Error(`${spec}: the registry gave no tarball (${String(pack.stderr).trim().split('\n').at(-1) ?? ''})`);
|
|
22
|
+
const untar = spawnSync('tar', ['-xzf', join(tmp, tarball), '-C', tmp], { encoding: 'utf8' });
|
|
23
|
+
if (untar.status !== 0) throw new Error(`${spec}: the tarball does not unpack (${String(untar.stderr).trim()})`);
|
|
24
|
+
const name = (JSON.parse(readFileSync(join(tmp, 'package', 'package.json'), 'utf8')) as { name: string }).name;
|
|
25
|
+
const dir = join(tmp, vendorOfPackage(name));
|
|
26
|
+
renameSync(join(tmp, 'package'), dir);
|
|
27
|
+
const lanes = readdirSync(dir).filter((l) => existsSync(join(dir, l, 'src', 'manifest.ts')) && !existsSync(join(dir, l, 'package.json'))).map((l) => join(dir, l));
|
|
28
|
+
const results = await p3Gate([dir, ...lanes]);
|
|
29
|
+
const unit = (u: string): string => u.slice(tmp.length + 1);
|
|
30
|
+
return {
|
|
31
|
+
spec,
|
|
32
|
+
failures: results.filter((r) => r.failures.length).map((r) => ({ unit: unit(r.unit), failures: r.failures })),
|
|
33
|
+
};
|
|
34
|
+
} finally {
|
|
35
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// One API published as several OpenAPI documents (Upstash publishes QStash's and Workflow's, both served on
|
|
2
|
+
// qstash-{region}.upstash.io with one key set, and sharing endpoints): the pack vendors each as published under
|
|
3
|
+
// `spec/openapi/<name>.{json,yaml}[.gz]`, patches address a document as `/documents/<name>/...`, and the documents are
|
|
4
|
+
// read as their union. A path and method, or a component, that two documents both give must be the same (their words,
|
|
5
|
+
// description and summary, aside); where they contradict each other, neither page is evidence against the other, and a
|
|
6
|
+
// patch rules which holds (citing what shows it: the vendor's own client, a recording), so the union is never a guess.
|
|
7
|
+
|
|
8
|
+
import { parse as parseYaml } from 'yaml';
|
|
9
|
+
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
|
10
|
+
import { join } from 'node:path';
|
|
11
|
+
import { gunzipSync } from 'node:zlib';
|
|
12
|
+
|
|
13
|
+
type Json = any;
|
|
14
|
+
export type SpecDocuments = { documents: Record<string, Json> };
|
|
15
|
+
|
|
16
|
+
/** The documents of `spec/openapi/`, by name, or undefined when the pack vendors one spec file. */
|
|
17
|
+
export function readSpecDocuments(specDir: string): SpecDocuments | undefined {
|
|
18
|
+
const dir = join(specDir, 'openapi');
|
|
19
|
+
if (!existsSync(dir)) return undefined;
|
|
20
|
+
const documents: Record<string, Json> = {};
|
|
21
|
+
for (const f of readdirSync(dir).filter((f) => /\.(json|ya?ml)(\.gz)?$/.test(f)).sort()) {
|
|
22
|
+
const bytes = readFileSync(join(dir, f));
|
|
23
|
+
const text = (f.endsWith('.gz') ? gunzipSync(bytes) : bytes).toString('utf8');
|
|
24
|
+
documents[f.replace(/\.(json|ya?ml)(\.gz)?$/, '')] = /\.ya?ml(\.gz)?$/.test(f) ? parseYaml(text) : JSON.parse(text);
|
|
25
|
+
}
|
|
26
|
+
if (Object.keys(documents).length < 2) throw new Error(`${dir}: one API in several documents needs at least two; a single document is spec/openapi.{json,yaml}`);
|
|
27
|
+
return { documents };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** A node without its words (description, summary, examples, and the tags a document files it under): what two
|
|
31
|
+
* documents must agree on. */
|
|
32
|
+
const substance = (n: Json): string => JSON.stringify(n, (k, v) => (['description', 'summary', 'example', 'examples', 'x-mint', 'tags'].includes(k) ? undefined : v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a.localeCompare(b))) : v));
|
|
33
|
+
|
|
34
|
+
/** The union of the documents as one OpenAPI document; each contradiction left unpatched is named, by pointer. */
|
|
35
|
+
export function mergeDocuments(docs: SpecDocuments): { doc: Json; contradictions: string[] } {
|
|
36
|
+
const names = Object.keys(docs.documents);
|
|
37
|
+
const first = docs.documents[names[0]!];
|
|
38
|
+
const doc: Json = { ...structuredClone(first), paths: {}, components: {} };
|
|
39
|
+
const from: Record<string, string> = {};
|
|
40
|
+
const contradictions: string[] = [];
|
|
41
|
+
const take = (target: Json, key: string, value: Json, pointer: string, name: string): void => {
|
|
42
|
+
if (target[key] === undefined) { target[key] = structuredClone(value); from[pointer] = name; return; }
|
|
43
|
+
if (substance(target[key]) !== substance(value)) contradictions.push(`/documents/${from[pointer]}${pointer} and /documents/${name}${pointer} differ`);
|
|
44
|
+
};
|
|
45
|
+
for (const name of names) {
|
|
46
|
+
const d = docs.documents[name];
|
|
47
|
+
if (substance(d.servers) !== substance(first.servers)) contradictions.push(`/documents/${name}/servers differs from /documents/${names[0]}/servers: not one API`);
|
|
48
|
+
for (const [path, item] of Object.entries((d.paths ?? {}) as Record<string, Json>)) {
|
|
49
|
+
const target = (doc.paths[path] ??= {});
|
|
50
|
+
for (const [k, v] of Object.entries(item)) take(target, k, v, `/paths/${path.replace(/~/g, '~0').replace(/\//g, '~1')}/${k}`, name);
|
|
51
|
+
}
|
|
52
|
+
for (const [part, entries] of Object.entries((d.components ?? {}) as Record<string, Record<string, Json>>)) {
|
|
53
|
+
const target = (doc.components[part] ??= {});
|
|
54
|
+
for (const [k, v] of Object.entries(entries)) take(target, k, v, `/components/${part}/${k}`, name);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return { doc, contradictions };
|
|
58
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// A vendor that publishes no spec of its API but ships its own client: the API as that client calls it. Tinybird
|
|
2
|
+
// publishes no OpenAPI of its platform API (its reference pages omit endpoints its own CLI calls: /v0/tags,
|
|
3
|
+
// /v0/sql_tables), and its Python client (`tinybird/client.py`, class TinyB, in tinybird-cli on PyPI) makes every
|
|
4
|
+
// request through `_req(url, method=..., data=...)`. Each such call, read from the client's syntax tree (the pack's
|
|
5
|
+
// `spec/extract-client-ops.py`), is vendored as data in `spec/client-ops.json`: its method, its URL as the client
|
|
6
|
+
// builds it (a label is the Python expression the client interpolates), the query keys it sends (literal, or the keys
|
|
7
|
+
// of the dict it urlencodes) and its body's keys, each typed by the client's own annotation of the value (`null`
|
|
8
|
+
// where the client does not say). A spec patch corrects the table as it corrects a JSON spec (derive-pack): what the
|
|
9
|
+
// client builds as a raw string (a token's scopes) comes from the vendor's reference page that way.
|
|
10
|
+
//
|
|
11
|
+
// The calls at one method and path are one operation, their query and body keys together. The client says nothing
|
|
12
|
+
// of what an answer holds, so an operation answers no resource here: its answers are the vendor's pages' examples.
|
|
13
|
+
|
|
14
|
+
import type { IrOperation, IrParam, SpecIR } from './spec-ir.ts';
|
|
15
|
+
|
|
16
|
+
export type ClientCall = {
|
|
17
|
+
/** the client method that makes the call, and its line */
|
|
18
|
+
client: string;
|
|
19
|
+
line: number;
|
|
20
|
+
method: string;
|
|
21
|
+
url: string;
|
|
22
|
+
query?: Record<string, string | null>;
|
|
23
|
+
body?: { encoding: 'json' | 'form' | 'raw'; fields?: Record<string, string | null>; passes?: string };
|
|
24
|
+
};
|
|
25
|
+
export type ClientOpsSpec = { format: 'client'; version: string; calls: ClientCall[] };
|
|
26
|
+
|
|
27
|
+
/** A label's name from the expression the client interpolates: `quote(datasource_name, safe='')` is
|
|
28
|
+
* `datasource_name`, `workspace['id']` is `workspace_id`. */
|
|
29
|
+
function labelOf(expr: string): string {
|
|
30
|
+
const inner = /^\w+\(\s*([\w[\]'"]+)/.exec(expr)?.[1] ?? expr;
|
|
31
|
+
return inner.replace(/\[['"](\w+)['"]\]/g, '_$1').replace(/\W+/g, '_');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The URL's path with each label named, no trailing slash (the client writes `/v0/user/workspaces/`). */
|
|
35
|
+
function pathOf(url: string): { path: string; labels: string[] } {
|
|
36
|
+
const raw = `/${url.split('?')[0]!.replace(/^\/+/, '')}`.replace(/(.)\/$/, '$1');
|
|
37
|
+
const labels: string[] = [];
|
|
38
|
+
const path = raw.replace(/\{([^}]+)\}/g, (_, expr: string) => { const name = labelOf(expr); labels.push(name); return `{${name}}`; });
|
|
39
|
+
return { path, labels };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const typeOf = (t: string | null | undefined): string => t ?? 'string';
|
|
43
|
+
|
|
44
|
+
export function fromClientCalls(spec: ClientOpsSpec): SpecIR {
|
|
45
|
+
const byRoute = new Map<string, { first: ClientCall; path: string; labels: string[]; calls: ClientCall[] }>();
|
|
46
|
+
for (const call of spec.calls) {
|
|
47
|
+
const { path, labels } = pathOf(call.url);
|
|
48
|
+
const key = `${call.method.toUpperCase()} ${path.replace(/\{[^}]+\}/g, '{}')}`;
|
|
49
|
+
const at = byRoute.get(key);
|
|
50
|
+
if (at) at.calls.push(call);
|
|
51
|
+
else byRoute.set(key, { first: call, path, labels, calls: [call] });
|
|
52
|
+
}
|
|
53
|
+
const operations: IrOperation[] = [...byRoute.values()].map(({ first, path, labels, calls }) => {
|
|
54
|
+
const query = new Map<string, IrParam>();
|
|
55
|
+
const body = new Map<string, IrParam>();
|
|
56
|
+
const encodings = new Set<string>();
|
|
57
|
+
for (const c of calls) {
|
|
58
|
+
for (const [name, t] of Object.entries(c.query ?? {})) if (!query.has(name)) query.set(name, { name, type: typeOf(t), required: false });
|
|
59
|
+
if (c.body) encodings.add(c.body.encoding);
|
|
60
|
+
for (const [name, t] of Object.entries(c.body?.fields ?? {})) if (!body.has(name)) body.set(name, { name, type: typeOf(t), required: false });
|
|
61
|
+
}
|
|
62
|
+
// a JSON body where any call sends one; a form where the client sends a dict; otherwise the body is the client's own bytes
|
|
63
|
+
const bodyEncoding: IrOperation['bodyEncoding'] = encodings.has('json') ? 'json' : encodings.has('form') ? 'form' : 'none';
|
|
64
|
+
return {
|
|
65
|
+
id: first.client,
|
|
66
|
+
// the client library that makes the call (`libnpmpublish.publish` is `libnpmpublish`): the operation's family
|
|
67
|
+
...(first.client.includes('.') ? { family: first.client.slice(0, first.client.indexOf('.')) } : {}),
|
|
68
|
+
method: first.method.toLowerCase(),
|
|
69
|
+
path,
|
|
70
|
+
pathParams: labels,
|
|
71
|
+
query: [...query.values()].sort((a, b) => a.name.localeCompare(b.name)),
|
|
72
|
+
body: [...body.values()].sort((a, b) => a.name.localeCompare(b.name)),
|
|
73
|
+
bodyEncoding,
|
|
74
|
+
successStatus: 200,
|
|
75
|
+
class: 'action',
|
|
76
|
+
};
|
|
77
|
+
});
|
|
78
|
+
// two routes whose first calls share a client method (one method calling twice) keep distinct ids
|
|
79
|
+
const seen = new Map<string, number>();
|
|
80
|
+
for (const o of operations.sort((a, b) => a.path.localeCompare(b.path) || a.method.localeCompare(b.method))) {
|
|
81
|
+
const n = (seen.get(o.id) ?? 0) + 1;
|
|
82
|
+
seen.set(o.id, n);
|
|
83
|
+
if (n > 1) o.id = `${o.id}_${n}`;
|
|
84
|
+
}
|
|
85
|
+
return { format: 'client', version: spec.version, resources: [], operations };
|
|
86
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// A command language's spec as data: the commands a client names in each request and what each takes and answers,
|
|
2
|
+
// read from the vendor's own command table. Redis publishes one file per command (redis/redis src/commands/<name>.json:
|
|
3
|
+
// its group, the version it arrived in, its arity, its arguments, its flags and its reply's schema); a pack whose wire
|
|
4
|
+
// carries Redis commands (Upstash's REST API: a command and its arguments in the path or the body) vendors the files of
|
|
5
|
+
// the commands its vendor serves and derives its command surface from them, as a line protocol's surface is its RFC's
|
|
6
|
+
// command table (spec-ir-lines.ts). A spec patch corrects the data as it corrects a JSON spec (derive-pack).
|
|
7
|
+
|
|
8
|
+
export type CommandArgument = { name: string; type: string; optional?: boolean; multiple?: boolean; token?: string; arguments?: CommandArgument[] };
|
|
9
|
+
/** A command: `id` as a client names it (`SET`, `CLIENT GETNAME` for a container's subcommand); `arity` as Redis gives
|
|
10
|
+
* it (n: exactly n words with the name; -n: at least n). */
|
|
11
|
+
export type CommandDef = { id: string; group: string; since: string; arity: number; summary: string; flags: string[]; arguments: CommandArgument[]; reply?: unknown };
|
|
12
|
+
export type CommandTableSpec = { format: 'command-table'; version: string; commands: CommandDef[] };
|
|
13
|
+
|
|
14
|
+
type RedisCommandFile = Record<string, {
|
|
15
|
+
summary?: string; group?: string; since?: string; arity?: number; container?: string;
|
|
16
|
+
command_flags?: string[]; arguments?: Array<Record<string, unknown>>; reply_schema?: unknown;
|
|
17
|
+
}>;
|
|
18
|
+
|
|
19
|
+
function argumentOf(a: Record<string, unknown>): CommandArgument {
|
|
20
|
+
return {
|
|
21
|
+
name: String(a.name),
|
|
22
|
+
type: String(a.type),
|
|
23
|
+
...(a.optional === true ? { optional: true } : {}),
|
|
24
|
+
...(a.multiple === true ? { multiple: true } : {}),
|
|
25
|
+
...(typeof a.token === 'string' ? { token: a.token } : {}),
|
|
26
|
+
...(Array.isArray(a.arguments) ? { arguments: (a.arguments as Array<Record<string, unknown>>).map(argumentOf) } : {}),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The command table of the vendored files (file name → its parsed JSON), in command order. */
|
|
31
|
+
export function fromRedisCommands(files: Record<string, RedisCommandFile>, version: string): CommandTableSpec {
|
|
32
|
+
const commands: CommandDef[] = [];
|
|
33
|
+
for (const [file, doc] of Object.entries(files)) {
|
|
34
|
+
for (const [name, c] of Object.entries(doc)) {
|
|
35
|
+
if (typeof c?.arity !== 'number') throw new Error(`spec-ir-commands: ${file}: ${name} gives no arity`);
|
|
36
|
+
commands.push({
|
|
37
|
+
id: c.container ? `${c.container} ${name}` : name,
|
|
38
|
+
group: String(c.group ?? ''),
|
|
39
|
+
since: String(c.since ?? ''),
|
|
40
|
+
arity: c.arity,
|
|
41
|
+
summary: String(c.summary ?? ''),
|
|
42
|
+
flags: c.command_flags ?? [],
|
|
43
|
+
arguments: (c.arguments ?? []).map(argumentOf),
|
|
44
|
+
...(c.reply_schema !== undefined ? { reply: c.reply_schema } : {}),
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
commands.sort((a, b) => a.id.localeCompare(b.id));
|
|
49
|
+
return { format: 'command-table', version, commands };
|
|
50
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// A Google API Discovery document as the IR (spec-ir.ts): the spec every Google REST API publishes at
|
|
2
|
+
// `https://<api>.googleapis.com/$discovery/rest?version=<v>` (https://developers.google.com/discovery/v1/reference/apis),
|
|
3
|
+
// and the one its own generated clients (googleapis, @googleapis/<api>) are built from. Google publishes no OpenAPI
|
|
4
|
+
// document for these APIs, so the Discovery document is the vendor's spec.
|
|
5
|
+
//
|
|
6
|
+
// The document is read into the OpenAPI 3 shape `fromOpenAPI` reads, as a Swagger 2 document is (fromSwagger2), with
|
|
7
|
+
// nothing vendor-specific beyond the Discovery format itself:
|
|
8
|
+
//
|
|
9
|
+
// - every method of every resource (recursively) is one operation: its `id` (`calendar.events.list`) is the
|
|
10
|
+
// operationId, its `httpMethod` and `path` (relative to `rootUrl` + `servicePath`, which becomes the server) are its
|
|
11
|
+
// route (a reserved expansion, `{+name}`, is read as a plain parameter: Calendar has none; an API whose path values hold
|
|
12
|
+
// slashes needs the `spanning` routes and is not yet read by this reader), its `path`-located parameters are path parameters and its `query`-located ones query parameters, a
|
|
13
|
+
// `repeated` parameter an array;
|
|
14
|
+
// - the document's own `parameters` (Google's standard query parameters: `fields`, `key`, `oauth_token`,
|
|
15
|
+
// `prettyPrint`, `quotaUser`, `alt`, `userIp`, https://cloud.google.com/apis/docs/system-parameters) are parameters
|
|
16
|
+
// of every operation, as every method takes them;
|
|
17
|
+
// - `request.$ref` is a JSON request body of that schema, and `response.$ref` the 200 answer; a method with no
|
|
18
|
+
// `response` answers 204 with no body (Google's REST answer to a delete or a stop);
|
|
19
|
+
// - `scopes` is the operation's security requirement (any one scope lets the call through), under the document's
|
|
20
|
+
// `auth.oauth2` scheme;
|
|
21
|
+
// - `schemas` are the component schemas, each `$ref: "Name"` rewritten to `#/components/schemas/Name`, and a
|
|
22
|
+
// Discovery `type: "any"` read as a schema of any type.
|
|
23
|
+
import { fromOpenAPI, type SpecIR } from './spec-ir.ts';
|
|
24
|
+
|
|
25
|
+
type Json = any;
|
|
26
|
+
|
|
27
|
+
/** A Discovery schema as an OpenAPI 3 schema. */
|
|
28
|
+
function schemaOf(node: Json): Json {
|
|
29
|
+
if (Array.isArray(node)) return node.map(schemaOf);
|
|
30
|
+
if (!node || typeof node !== 'object') return node;
|
|
31
|
+
const out: Json = {};
|
|
32
|
+
for (const [k, v] of Object.entries(node)) {
|
|
33
|
+
if (k === '$ref' && typeof v === 'string') out.$ref = `#/components/schemas/${v}`;
|
|
34
|
+
else if (k === 'type' && v === 'any') continue;
|
|
35
|
+
// a schema's own `id` names it in the Discovery format; OpenAPI names it by its key
|
|
36
|
+
else if (k === 'id' && typeof v === 'string' && typeof node.type === 'string' && !('location' in node)) continue;
|
|
37
|
+
else if (k === 'properties' && v && typeof v === 'object') out.properties = Object.fromEntries(Object.entries(v).map(([p, s]) => [p, schemaOf(s)]));
|
|
38
|
+
else if (k === 'items' || k === 'additionalProperties') out[k] = schemaOf(v);
|
|
39
|
+
else if (k === 'annotations' || k === 'enumDescriptions' || k === 'location' || k === 'repeated') continue;
|
|
40
|
+
else out[k] = v;
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** A Discovery parameter as an OpenAPI 3 parameter. */
|
|
46
|
+
function parameterOf(name: string, p: Json): Json {
|
|
47
|
+
const scalar = schemaOf({ ...p, description: undefined });
|
|
48
|
+
delete scalar.description;
|
|
49
|
+
delete scalar.required;
|
|
50
|
+
const schema = p.repeated ? { type: 'array', items: scalar } : scalar;
|
|
51
|
+
return { name, in: p.location === 'path' ? 'path' : 'query', ...(p.required || p.location === 'path' ? { required: true } : {}), ...(p.description ? { description: p.description } : {}), schema };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Every method of a Discovery resource tree, depth first, in the document's order. */
|
|
55
|
+
export function discoveryMethods(resources: Json): Json[] {
|
|
56
|
+
const out: Json[] = [];
|
|
57
|
+
for (const r of Object.values(resources ?? {}) as Json[]) {
|
|
58
|
+
for (const m of Object.values(r?.methods ?? {}) as Json[]) out.push(m);
|
|
59
|
+
out.push(...discoveryMethods(r?.resources));
|
|
60
|
+
}
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** A Google API Discovery document as an OpenAPI 3 document (the shape `fromOpenAPI` reads). */
|
|
65
|
+
export function discoveryToOpenAPI(doc: Json): Json {
|
|
66
|
+
const global = (doc.parameters ?? {}) as Record<string, Json>;
|
|
67
|
+
const paths: Record<string, Record<string, Json>> = {};
|
|
68
|
+
for (const m of discoveryMethods(doc.resources)) {
|
|
69
|
+
const path = `/${String(m.path).replace(/^\//, '').replace(/\{\+/g, '{')}`;
|
|
70
|
+
const own = Object.entries((m.parameters ?? {}) as Record<string, Json>);
|
|
71
|
+
const parameters = [
|
|
72
|
+
...own.filter(([, p]) => p.location === 'path').map(([n, p]) => parameterOf(n, p)),
|
|
73
|
+
...own.filter(([, p]) => p.location === 'query').map(([n, p]) => parameterOf(n, p)),
|
|
74
|
+
...Object.entries(global).filter(([n]) => !own.some(([o]) => o === n)).map(([n, p]) => parameterOf(n, p)),
|
|
75
|
+
];
|
|
76
|
+
const response = m.response?.$ref ? { 200: { description: 'Successful response', content: { 'application/json': { schema: { $ref: `#/components/schemas/${m.response.$ref}` } } } } } : { 204: { description: 'Successful response with no body' } };
|
|
77
|
+
(paths[path] ??= {})[String(m.httpMethod).toLowerCase()] = {
|
|
78
|
+
operationId: m.id,
|
|
79
|
+
...(m.description ? { description: m.description } : {}),
|
|
80
|
+
parameters,
|
|
81
|
+
...(m.request?.$ref ? { requestBody: { content: { 'application/json': { schema: { $ref: `#/components/schemas/${m.request.$ref}` } } } } } : {}),
|
|
82
|
+
responses: response,
|
|
83
|
+
...(Array.isArray(m.scopes) && m.scopes.length ? { security: [{ Oauth2: m.scopes }] } : {}),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
openapi: '3.0.0',
|
|
88
|
+
info: { title: doc.title ?? doc.name, version: doc.version, description: doc.description, 'x-revision': doc.revision },
|
|
89
|
+
servers: [{ url: `${String(doc.rootUrl ?? '').replace(/\/+$/, '')}/${String(doc.servicePath ?? '').replace(/^\/+/, '')}`.replace(/\/+$/, '') }],
|
|
90
|
+
components: {
|
|
91
|
+
schemas: Object.fromEntries(Object.entries((doc.schemas ?? {}) as Record<string, Json>).map(([n, s]) => [n, schemaOf(s)])),
|
|
92
|
+
...(doc.auth?.oauth2 ? { securitySchemes: { Oauth2: { type: 'oauth2', flows: { authorizationCode: { authorizationUrl: 'https://accounts.google.com/o/oauth2/auth', tokenUrl: 'https://oauth2.googleapis.com/token', scopes: Object.fromEntries(Object.entries(doc.auth.oauth2.scopes ?? {}).map(([s, d]: [string, Json]) => [s, d?.description ?? ''])) } } } } } : {}),
|
|
93
|
+
},
|
|
94
|
+
paths,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Is this a Google API Discovery document? */
|
|
99
|
+
export const isDiscovery = (doc: Json): boolean => doc?.kind === 'discovery#restDescription';
|
|
100
|
+
|
|
101
|
+
/** A Google API Discovery document as the IR. */
|
|
102
|
+
export function fromDiscovery(doc: Json): SpecIR {
|
|
103
|
+
return fromOpenAPI(discoveryToOpenAPI(doc));
|
|
104
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// THE SPEC IR's GraphQL front end: a vendor's published SDL as operations the derived-pack generator
|
|
2
|
+
// reads (docs/contributing/architecture.md, "Protocol 3"). A GraphQL API is one endpoint whose
|
|
3
|
+
// operations are its root fields: each Query field is a read, each Mutation field a write. Resources
|
|
4
|
+
// are the object types a caller can refetch (those implementing `Node`, or carrying an `id: ID!`).
|
|
5
|
+
// Kept apart from spec-ir.ts because it needs the `graphql` protocol library.
|
|
6
|
+
import { buildSchema, getNamedType, isListType, isNonNullType, isObjectType, type GraphQLField, type GraphQLObjectType, type GraphQLOutputType } from 'graphql';
|
|
7
|
+
import type { OperationClass } from './spec-ir.ts';
|
|
8
|
+
|
|
9
|
+
export type GraphqlOperation = {
|
|
10
|
+
/** `query.<field>` or `mutation.<field>` */
|
|
11
|
+
id: string;
|
|
12
|
+
kind: 'query' | 'mutation';
|
|
13
|
+
field: string;
|
|
14
|
+
args: Array<{ name: string; type: string; required: boolean }>;
|
|
15
|
+
/** the named type the field returns, and whether it is a list or a connection of it */
|
|
16
|
+
returns: string;
|
|
17
|
+
class: OperationClass;
|
|
18
|
+
resource?: string;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export type GraphqlIR = { format: 'graphql'; resources: Array<{ name: string; fields: string[] }>; operations: GraphqlOperation[] };
|
|
22
|
+
|
|
23
|
+
const unwrap = (t: GraphQLOutputType): GraphQLOutputType => (isNonNullType(t) ? unwrap(t.ofType) : t);
|
|
24
|
+
|
|
25
|
+
export function fromGraphQL(sdl: string): GraphqlIR {
|
|
26
|
+
const schema = buildSchema(sdl, { assumeValidSDL: true });
|
|
27
|
+
const isResource = (t: GraphQLObjectType): boolean => t.getInterfaces().some((i) => i.name === 'Node') || !!t.getFields().id;
|
|
28
|
+
const resources = Object.values(schema.getTypeMap())
|
|
29
|
+
.filter((t): t is GraphQLObjectType => isObjectType(t) && !t.name.startsWith('__') && isResource(t))
|
|
30
|
+
.map((t) => ({ name: t.name, fields: Object.keys(t.getFields()).sort() }))
|
|
31
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
32
|
+
const names = new Set(resources.map((r) => r.name));
|
|
33
|
+
const connectionOf = (t: GraphQLObjectType): string | undefined => {
|
|
34
|
+
const nodes = t.getFields().nodes;
|
|
35
|
+
return t.name.endsWith('Connection') && nodes ? getNamedType(nodes.type).name : undefined;
|
|
36
|
+
};
|
|
37
|
+
const operations: GraphqlOperation[] = [];
|
|
38
|
+
for (const [kind, root] of [['query', schema.getQueryType()], ['mutation', schema.getMutationType()]] as const) {
|
|
39
|
+
for (const f of Object.values(root?.getFields() ?? {}) as Array<GraphQLField<unknown, unknown>>) {
|
|
40
|
+
const bare = unwrap(f.type);
|
|
41
|
+
const named = getNamedType(f.type);
|
|
42
|
+
const connection = isObjectType(named) ? connectionOf(named) : undefined;
|
|
43
|
+
const list = isListType(bare) || !!connection;
|
|
44
|
+
const target = connection ?? named.name;
|
|
45
|
+
const resource = names.has(target) ? target : undefined;
|
|
46
|
+
const verb = f.name.toLowerCase();
|
|
47
|
+
const cls: OperationClass =
|
|
48
|
+
kind === 'query'
|
|
49
|
+
? list ? 'list' : resource ? 'retrieve' : 'computed'
|
|
50
|
+
: /^(create|add)/.test(verb) ? 'create' : /^(update|edit|set)/.test(verb) ? 'update' : /^(delete|remove)/.test(verb) ? 'delete' : 'action';
|
|
51
|
+
operations.push({
|
|
52
|
+
id: `${kind}.${f.name}`,
|
|
53
|
+
kind,
|
|
54
|
+
field: f.name,
|
|
55
|
+
args: f.args.map((a) => ({ name: a.name, type: String(a.type), required: isNonNullType(a.type) })).sort((a, b) => a.name.localeCompare(b.name)),
|
|
56
|
+
returns: target,
|
|
57
|
+
class: cls,
|
|
58
|
+
...(resource ? { resource } : {}),
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
operations.sort((a, b) => a.id.localeCompare(b.id));
|
|
63
|
+
return { format: 'graphql', resources, operations };
|
|
64
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// A line protocol's spec as data: the commands a client sends and the replies each may be answered,
|
|
2
|
+
// read from the RFC's own table of them. The IR a byte-stream pack derives its surface from, as an HTTP
|
|
3
|
+
// pack derives its from an OpenAPI document; a spec patch corrects it the same way (derive-pack).
|
|
4
|
+
//
|
|
5
|
+
// SMTP's table is RFC 5321 §4.3.2 "Command-Reply Sequences": each command, then its intermediate (I),
|
|
6
|
+
// success (S) and error (E) codes, where `I: 354 -> data -> S: 250` says what answers the lines that
|
|
7
|
+
// follow an intermediate reply. Every command may also answer the section's "any SMTP command" codes.
|
|
8
|
+
|
|
9
|
+
export type LineReplies = { intermediate: number[]; success: number[]; error: number[] };
|
|
10
|
+
/** A command and its replies; `then`, what answers the lines after an intermediate reply (DATA's payload,
|
|
11
|
+
* a SASL exchange). */
|
|
12
|
+
export type LineCommand = LineReplies & { id: string; then?: LineReplies };
|
|
13
|
+
export type LineSpec = { format: 'command-reply'; version: string; anyCommand: number[]; commands: LineCommand[] };
|
|
14
|
+
|
|
15
|
+
const codes = (text: string): number[] => [...new Set((text.replace(/\([^)]*\)/g, '').match(/\b[2-5]\d\d\b/g) ?? []).map(Number))];
|
|
16
|
+
|
|
17
|
+
/** The replies in one level of a command's entry: `S: 250 E: 552, 451` and the like. */
|
|
18
|
+
function replies(text: string): LineReplies {
|
|
19
|
+
const out: LineReplies = { intermediate: [], success: [], error: [] };
|
|
20
|
+
// `S: 250 E: 552, 451` splits into its letters and what follows each
|
|
21
|
+
const parts = text.replace(/\([^)]*\)/g, '').split(/\b([ISE]):/);
|
|
22
|
+
for (let i = 1; i < parts.length; i += 2) {
|
|
23
|
+
const key = parts[i] === 'I' ? 'intermediate' : parts[i] === 'S' ? 'success' : 'error';
|
|
24
|
+
out[key] = [...new Set([...out[key], ...codes(parts[i + 1]!)])];
|
|
25
|
+
}
|
|
26
|
+
return out;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The command-reply table of an RFC in RFC 5321's form, from the RFC's text. */
|
|
30
|
+
export function fromCommandReplies(text: string, section = '4.3.2'): LineSpec {
|
|
31
|
+
const lines = text.split('\n')
|
|
32
|
+
// a page's footer and the next page's header are the RFC's pagination, not its words
|
|
33
|
+
.filter((l) => !/\[Page \d+\]\s*$/.test(l) && !/^RFC \d+\s{2,}/.test(l) && !l.includes('\f'));
|
|
34
|
+
const start = lines.findIndex((l) => l.startsWith(`${section}. `));
|
|
35
|
+
if (start < 0) throw new Error(`spec-ir-lines: the text has no section ${section}`);
|
|
36
|
+
const end = lines.findIndex((l, i) => i > start && /^\d+(\.\d+)*\.\s{2}/.test(l));
|
|
37
|
+
const body = lines.slice(start + 1, end < 0 ? undefined : end);
|
|
38
|
+
const version = /^Request for Comments:\s*(\d+)/m.exec(text)?.[1];
|
|
39
|
+
// the codes any command may answer: the list the section gives before its specific sequences
|
|
40
|
+
const specific = body.findIndex((l) => /Specific sequences are:/.test(l));
|
|
41
|
+
const anyCommand = body.slice(0, specific).filter((l) => /^\s{3}\d{3}\s{2}/.test(l)).map((l) => Number(l.trim().slice(0, 3)));
|
|
42
|
+
const commands: LineCommand[] = [];
|
|
43
|
+
let entry: { names: string[]; own: string[]; data: string[] } | undefined;
|
|
44
|
+
let deep = false;
|
|
45
|
+
const close = () => {
|
|
46
|
+
if (!entry) return;
|
|
47
|
+
const own = replies(entry.own.join(' '));
|
|
48
|
+
const then = entry.data.length ? replies(`S:${entry.data.join(' ')}`) : undefined;
|
|
49
|
+
for (const id of entry.names) commands.push({ id, ...own, ...(then ? { then } : {}) });
|
|
50
|
+
};
|
|
51
|
+
for (const line of body.slice(specific + 1)) {
|
|
52
|
+
if (!line.trim()) continue;
|
|
53
|
+
const indent = line.length - line.trimStart().length;
|
|
54
|
+
// a command's heading is its name in capitals, one level in ("EHLO or HELO", "CONNECTION ESTABLISHMENT")
|
|
55
|
+
if (indent <= 6 && /^[A-Z]+(?: (?:or )?[A-Z]+)*$/.test(line.trim())) {
|
|
56
|
+
close();
|
|
57
|
+
entry = { names: line.trim().split(' or '), own: [], data: [] };
|
|
58
|
+
deep = false;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (!entry) continue;
|
|
62
|
+
const arrow = line.lastIndexOf('->');
|
|
63
|
+
if (arrow >= 0) {
|
|
64
|
+
// `I: 354 -> data -> S: 250`: the first part is the command's own, what follows the last arrow answers the data
|
|
65
|
+
entry.own.push(line.slice(0, line.indexOf('->')));
|
|
66
|
+
entry.data.push(line.slice(arrow + 2).replace(/^\s*S:/, ''));
|
|
67
|
+
deep = true;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
// what is indented past the command's own replies continues what answers the data
|
|
71
|
+
if (deep && indent > 12) entry.data.push(line);
|
|
72
|
+
else { deep = false; entry.own.push(line); }
|
|
73
|
+
}
|
|
74
|
+
close();
|
|
75
|
+
return { format: 'command-reply', version: version ? `RFC ${version}` : section, anyCommand, commands };
|
|
76
|
+
}
|