@pepys/kit 0.0.0-stage → 0.0.1

package/src/generators/format-row-readers.ts

@@ -0,0 +1,116 @@
+import { match } from 'ts-pattern';
+import type { ConsumerDeclaration } from '../consumers/types.ts';
+import type { EntityKind } from '../link-types/types.ts';
+import type { ColumnDeclaration } from '../types.ts';
+import { collectViewKinds } from './collect-view-kinds.ts';
+import { formatLiteral } from './format-literal.ts';
+import { formatPropertyAccess } from './format-property-access.ts';
+import { formatPropertyName } from './format-property-name.ts';
+import { toEntityIDTypeName } from './to-entity-id-type-name.ts';
+import { toPascalCase } from './to-pascal-case.ts';
+
+// One reader per table in `reads`, through the two host functions of the table: the length of the
+// row's JSON, or -1 for no row, and a copy of it into a buffer.
+export function formatRowReaders(consumer: ConsumerDeclaration): string[] {
+  if (consumer.reads.length === 0) {
+    return [];
+  }
+
+  const viewKinds = collectViewKinds(consumer);
+
+  return [
+    ...consumer.reads.flatMap((tableName) => {
+      const table = consumer.tables[tableName];
+
+      return table === undefined
+        ? []
+        : [formatRowReader(tableName, Object.entries(table.columns), viewKinds)];
+    }),
+    [
+      'function isEntityID(value: unknown): value is EntityID {',
+      "  return typeof value === 'string';",
+      '}',
+    ].join('\n'),
+  ];
+}
+
+function formatRowReader(
+  tableName: string,
+  columns: readonly (readonly [string, ColumnDeclaration])[],
+  viewKinds: ReadonlyMap<string, EntityKind>,
+): string {
+  const name = toPascalCase(tableName);
+
+  const checks = columns.flatMap(([columnName, column]) =>
+    formatCheck(columnName, column, viewKinds.get(`${tableName}.${columnName}`)),
+  );
+
+  const fields = columns.map(
+    ([columnName]) =>
+      `    ${formatPropertyName(columnName)}: ${formatPropertyAccess('parsed', columnName)},`,
+  );
+
+  return [
+    `declare function read${name}Length(key: string): number;`,
+    `declare function read${name}Into(key: string, row: Uint8Array): void;`,
+    '',
+    `export function read${name}Row(key: string): ${name}Row | undefined {`,
+    `  const length = read${name}Length(key);`,
+    '',
+    '  if (length < 0) {',
+    '    return undefined;',
+    '  }',
+    '',
+    '  const row = new Uint8Array(length);',
+    '',
+    `  read${name}Into(key, row);`,
+    '',
+    '  const parsed: unknown = JSON.parse(new TextDecoder().decode(row));',
+    '',
+    "  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {",
+    '    return undefined;',
+    '  }',
+    '',
+    ...checks,
+    '  return {',
+    ...fields,
+    '  };',
+    '}',
+  ].join('\n');
+}
+
+function formatCheck(
+  columnName: string,
+  column: ColumnDeclaration,
+  kind: EntityKind | undefined,
+): string[] {
+  const value = formatPropertyAccess('parsed', columnName);
+  const mismatch = formatMismatch(value, column, kind);
+  const conditions = [`!(${formatLiteral(columnName)} in parsed)`, ...mismatch];
+
+  return [`  if (${conditions.join(' || ')}) {`, '    return undefined;', '  }', ''];
+}
+
+function formatMismatch(
+  value: string,
+  column: ColumnDeclaration,
+  kind: EntityKind | undefined,
+): string[] {
+  const mismatch = match(column.kind)
+    .with('text', 'timestamp', () => `typeof ${value} !== 'string'`)
+    .with('integer', 'real', () => `typeof ${value} !== 'number'`)
+    .with('boolean', () => `typeof ${value} !== 'boolean'`)
+    .with('entityID', () =>
+      kind === undefined
+        ? `!isEntityID(${value})`
+        : `typeof ${value} !== 'string' || !is${toEntityIDTypeName(kind)}(${value})`,
+    )
+    .with('json', () => null)
+    .exhaustive();
+
+  if (mismatch === null) {
+    return [];
+  }
+
+  return [column.nullable === true ? `(${value} !== null && (${mismatch}))` : `(${mismatch})`];
+}

package/src/generators/generate-boundary-glue.ts

@@ -0,0 +1,60 @@
+import type { ConsumerDeclaration } from '../consumers/types.ts';
+import type { EntityKind } from '../link-types/types.ts';
+import type { FactFamily } from '../sources/types.ts';
+import { formatEntityIDs } from './format-entity-ids.ts';
+import { formatFactReader } from './format-fact-reader.ts';
+import { formatModelContract } from './format-model-contract.ts';
+import { formatOperations } from './format-operations.ts';
+import { formatRowReaders } from './format-row-readers.ts';
+import { toPascalCase } from './to-pascal-case.ts';
+
+export interface BoundaryGlueParameters {
+  consumer: ConsumerDeclaration;
+
+  // The families of the deployment. The glue types the facts of those on the consumer's streams.
+  families: readonly FactFamily[];
+
+  // the plug-in's checked settings, or undefined when it declares none
+  settings: Readonly<Record<string, string | number | boolean>> | undefined;
+}
+
+// The one file that a consumer's entry imports: entity IDs, the settings, the model contract, the
+// facts it reads, its rows and its operations. The glue reads a row of the consumer's own tables
+// only, and runs no zod.
+export function generateBoundaryGlue(parameters: BoundaryGlueParameters): string {
+  const consumer = parameters.consumer;
+  const consumerName = toPascalCase(consumer.name);
+
+  const streams = new Set<string>(consumer.input.streams);
+
+  const families = parameters.families.filter((family) => streams.has(family.stream));
+
+  const sections = [
+    ...formatEntityIDs(collectEntityKinds(consumer, families)),
+    ...(parameters.settings === undefined ? [] : [formatSettings(parameters.settings)]),
+    formatModelContract(consumerName),
+    ...formatFactReader({ consumerName, input: consumer.input, families }),
+    ...formatOperations(consumer),
+    ...formatRowReaders(consumer),
+  ];
+
+  return `${sections.join('\n\n')}\n`;
+}
+
+function collectEntityKinds(
+  consumer: ConsumerDeclaration,
+  families: readonly FactFamily[],
+): EntityKind[] {
+  const kinds = new Set<EntityKind>([
+    ...(consumer.input.mode === 'scope' ? [consumer.input.scopeKind] : []),
+    ...families.map((family) => family.entityKind),
+    ...consumer.entityViews.map((view) => view.kind),
+    ...consumer.links.flatMap((link) => [...link.from, ...link.to]),
+  ]);
+
+  return [...kinds].toSorted();
+}
+
+function formatSettings(settings: Readonly<Record<string, string | number | boolean>>): string {
+  return `export const settings = ${JSON.stringify(settings, null, 2)} as const;`;
+}

package/src/generators/generate-sqlite-schema.ts

@@ -0,0 +1,78 @@
+import { match } from 'ts-pattern';
+import type { ConsumerDeclaration, EntityViewDeclaration } from '../consumers/types.ts';
+import type { ColumnDeclaration } from '../types.ts';
+import { quoteSqlIdentifier } from './quote-sql-identifier.ts';
+
+export function generateSqliteSchema(consumer: ConsumerDeclaration): string {
+  const statements: string[] = [];
+
+  for (const [name, table] of Object.entries(consumer.tables)) {
+    statements.push(formatCreateTable(name, table.columns, table.primaryKey));
+
+    for (const index of table.indexes) {
+      const indexName = quoteSqlIdentifier(`${name}_${index.columns.join('_')}_idx`);
+      const columns = index.columns.map((column) => quoteSqlIdentifier(column)).join(', ');
+
+      statements.push(`CREATE INDEX ${indexName} ON ${quoteSqlIdentifier(name)} (${columns});`);
+    }
+  }
+
+  if (consumer.entityViews.length > 0) {
+    statements.push(formatEntityView(consumer.entityViews));
+  }
+
+  return `${statements.join('\n\n')}\n`;
+}
+
+function formatCreateTable(
+  name: string,
+  columns: Readonly<Record<string, ColumnDeclaration>>,
+  primaryKey: readonly string[],
+): string {
+  const definitions = Object.entries(columns).map(
+    ([columnName, column]) =>
+      `  ${quoteSqlIdentifier(columnName)} ${toSqliteType(column)}${column.nullable === true ? '' : ' NOT NULL'}`,
+  );
+
+  return [
+    `CREATE TABLE ${quoteSqlIdentifier(name)} (`,
+    [
+      ...definitions,
+      `  PRIMARY KEY (${primaryKey.map((column) => quoteSqlIdentifier(column)).join(', ')})`,
+    ].join(',\n'),
+    ');',
+  ].join('\n');
+}
+
+function toSqliteType(column: ColumnDeclaration): string {
+  return match(column.kind)
+    .with('text', 'json', 'entityID', 'timestamp', () => 'TEXT')
+    .with('integer', 'boolean', () => 'INTEGER')
+    .with('real', () => 'REAL')
+    .exhaustive();
+}
+
+function formatEntityView(views: readonly EntityViewDeclaration[]): string {
+  const selects = views.map((view) => formatEntitySelect(view)).join('\nUNION ALL\n');
+
+  return `CREATE VIEW "entity" AS\n${selects};`;
+}
+
+function formatEntitySelect(view: EntityViewDeclaration): string {
+  const kind = `'${view.kind.replaceAll("'", "''")}'`;
+
+  const selections = [
+    `${quoteSqlIdentifier(view.id)} AS "id"`,
+    `${kind} AS "kind"`,
+    `${quoteSqlIdentifier(view.title)} AS "title"`,
+    `${formatOptionalColumn(view.url)} AS "url"`,
+    `${formatOptionalColumn(view.actor)} AS "actor"`,
+    `${quoteSqlIdentifier(view.updatedAt)} AS "updated_at"`,
+  ].map((selection) => `  ${selection}`);
+
+  return ['SELECT', selections.join(',\n'), `FROM ${quoteSqlIdentifier(view.table)}`].join('\n');
+}
+
+function formatOptionalColumn(column: string | null): string {
+  return column === null ? 'NULL' : quoteSqlIdentifier(column);
+}

package/src/generators/is-identifier.ts

@@ -0,0 +1,5 @@
+const identifierPattern = /^[A-Za-z_$][\w$]*$/u;
+
+export function isIdentifier(name: string): boolean {
+  return identifierPattern.test(name);
+}

package/src/generators/quote-sql-identifier.ts

@@ -0,0 +1,3 @@
+export function quoteSqlIdentifier(name: string): string {
+  return `"${name.replaceAll('"', '""')}"`;
+}

package/src/generators/to-entity-id-type-name.ts

@@ -0,0 +1,6 @@
+import type { EntityKind } from '../link-types/types.ts';
+import { toPascalCase } from './to-pascal-case.ts';
+
+export function toEntityIDTypeName(kind: EntityKind): string {
+  return `${toPascalCase(kind)}ID`;
+}

package/src/generators/to-pascal-case.ts

@@ -0,0 +1,7 @@
+export function toPascalCase(name: string): string {
+  return name
+    .split(/[^A-Za-z0-9]+/u)
+    .filter((word) => word !== '')
+    .map((word) => word.slice(0, 1).toUpperCase() + word.slice(1))
+    .join('');
+}

package/src/generators.ts

@@ -0,0 +1,4 @@
+export { generateBoundaryGlue } from './generators/generate-boundary-glue.ts';
+export type { BoundaryGlueParameters } from './generators/generate-boundary-glue.ts';
+export { generateSqliteSchema } from './generators/generate-sqlite-schema.ts';
+export { toPascalCase } from './generators/to-pascal-case.ts';

package/src/ids/types.ts

@@ -0,0 +1,25 @@
+import type { Tagged } from 'type-fest';
+
+export type EntityID = Tagged<string, 'EntityID'>;
+
+export type FactID = Tagged<string, 'FactID'>;
+
+export type StreamName = Tagged<string, 'StreamName'>;
+
+export type StreamOffset = Tagged<number, 'StreamOffset'>;
+
+export type ScopeID = Tagged<string, 'ScopeID'>;
+
+export type BlobHash = Tagged<string, 'BlobHash'>;
+
+export type VersionHash = Tagged<string, 'VersionHash'>;
+
+export type Reference = Tagged<string, 'Reference'>;
+
+export type BucketNumber = Tagged<number, 'BucketNumber'>;
+
+export type ClientID = Tagged<string, 'ClientID'>;
+
+export type GrantID = Tagged<string, 'GrantID'>;
+
+export type RequestID = Tagged<string, 'RequestID'>;

package/src/index.ts

@@ -0,0 +1,36 @@
+export { buildEnrichmentFactID } from './facts/build-enrichment-fact-id.ts';
+export { buildEntityID } from './entity-ids/build-entity-id.ts';
+export { buildEventFactID } from './facts/build-event-fact-id.ts';
+export { buildManualFactID } from './facts/build-manual-fact-id.ts';
+export { buildObservationFactID } from './facts/build-observation-fact-id.ts';
+export { buildWebhookFactID } from './facts/build-webhook-fact-id.ts';
+export { checkDeployment } from './deployments/check-deployment.ts';
+export { defineConsumer } from './consumers/define-consumer.ts';
+export { defineDeployment } from './deployments/define-deployment.ts';
+export { defineFactFamily } from './sources/define-fact-family.ts';
+export { defineLinkType } from './link-types/define-link-type.ts';
+export { definePlugin } from './deployments/define-plugin.ts';
+export { defineQuery } from './queries/define-query.ts';
+export { defineSource } from './sources/define-source.ts';
+export { defineTable } from './tables/define-table.ts';
+export { findLinkProblem } from './links/find-link-problem.ts';
+export { isDeliveryAuthentic } from './delivery-checks/is-delivery-authentic.ts';
+export type { EntityIDFailure } from './entity-ids/entity-id-failure.ts';
+export type { EntityIDParts } from './entity-ids/entity-id-parts.ts';
+export { listReceiveRoutes } from './deployments/list-receive-routes.ts';
+export { parseEntityID } from './entity-ids/parse-entity-id.ts';
+export { parseFactEnvelope } from './facts/parse-fact-envelope.ts';
+export { parseStreamName } from './facts/parse-stream-name.ts';
+export { readFactBatch } from './sources/read-fact-batch.ts';
+export { receiveFactBatch } from './sources/receive-fact-batch.ts';
+export type { TableRow } from './tables/table-row.ts';
+export type * from './consumers/types.ts';
+export type * from './deployments/types.ts';
+export type * from './ids/types.ts';
+export type * from './link-types/types.ts';
+export type * from './links/types.ts';
+export type * from './send/types.ts';
+export type * from './queries/types.ts';
+export type * from './sources/types.ts';
+export type * from './store-reader/store-reader.ts';
+export type * from './types.ts';

package/src/link-types/define-link-type.ts

@@ -0,0 +1,9 @@
+import type { EntityKind, LinkTypeDeclaration } from './types.ts';
+
+export function defineLinkType<
+  const Name extends string,
+  const From extends EntityKind,
+  const To extends EntityKind,
+>(linkType: LinkTypeDeclaration<Name, From, To>): LinkTypeDeclaration<Name, From, To> {
+  return linkType;
+}

package/src/link-types/types.ts

@@ -0,0 +1,15 @@
+// `<source>:<kind>`, in an entity view, a link end, a scope kind and an entity ID alike.
+export type EntityKind = `${string}:${string}`;
+
+export interface LinkTypeDeclaration<
+  Name extends string = string,
+  From extends EntityKind = EntityKind,
+  To extends EntityKind = EntityKind,
+> {
+  readonly name: Name;
+  readonly from: readonly From[];
+  readonly to: readonly To[];
+
+  // `neighbourhood` follows a hub link type only when the caller asks
+  readonly hub: boolean;
+}

package/src/links/find-link-problem.ts

@@ -0,0 +1,53 @@
+import { parseEntityID } from '../entity-ids/parse-entity-id.ts';
+import type { CheckedLink, LinkCheckConsumer, LinkProblem } from './types.ts';
+
+// The link check, from the consumer's declaration alone: it looks no row up, so a link to an
+// entity that has not arrived passes. It reports the first problem in this order, which the host's
+// Rust check keeps: the type, the `from` end, the `to` end, the confidence, the evidence.
+export function findLinkProblem(
+  consumer: LinkCheckConsumer,
+  link: CheckedLink,
+): LinkProblem | undefined {
+  const linkType = consumer.links.find((declared) => declared.name === link.type);
+
+  if (linkType === undefined) {
+    return { kind: 'not-declared', type: link.type, consumer: consumer.name };
+  }
+
+  const endProblem =
+    findEndProblem('from', link.from, linkType.from) ?? findEndProblem('to', link.to, linkType.to);
+
+  if (endProblem !== undefined) {
+    return endProblem;
+  }
+
+  if (!(link.confidence >= 0 && link.confidence <= 1)) {
+    return { kind: 'bad-confidence', confidence: link.confidence };
+  }
+
+  if (link.evidence === '') {
+    return { kind: 'no-evidence' };
+  }
+
+  return undefined;
+}
+
+function findEndProblem(
+  end: 'from' | 'to',
+  id: string,
+  allowed: readonly string[],
+): LinkProblem | undefined {
+  const parsed = parseEntityID(id);
+
+  if (!parsed.ok) {
+    return { kind: 'not-an-entity-id', end, id };
+  }
+
+  const kind = `${parsed.value.source}:${parsed.value.kind}`;
+
+  if (!allowed.some((allowedKind) => allowedKind === kind)) {
+    return { kind: 'wrong-kind', end, id, allowed };
+  }
+
+  return undefined;
+}