ignotum 0.0.13 → 0.0.15

package/src/dev-runtime/database.ts

@@ -43,10 +43,15 @@ import type {
   RuntimeTableDefinition,
 } from "@ignotum/contracts/runtime/schema";
 import {
+  applyValueDefaults,
+  descriptorFields,
+  evolveStoredFields,
   getValueDescriptor,
   SchemaDefinitionTypeId,
   type ValueDescriptor,
 } from "@ignotum/contracts/schema";
+import { SchemaSnapshot } from "@ignotum/contracts/version";
+import { encodeCanonical } from "@ignotum/versions";
 import type { FileValue } from "@ignotum/contracts/schema/file";
 
 import { DevelopmentDatabase } from "./migrations.js";
@@ -272,9 +277,13 @@ const encodeFields = Effect.fn("LocalDatabase.encodeFields")(function* (
   operation: Extract<DatabaseOperation, "insert" | "patch" | "replace">,
   id: string | null,
 ) {
-  return yield* Schema.encodeEffect(fieldsCodec(table), {
+  const normalized = yield* Effect.try({
+    try: () => (operation === "patch" ? value : applyValueDefaults(table.schema, value)),
+    catch: (cause) => schemaMismatch(tableName, id, operation, cause),
+  });
+  return yield* Schema.encodeUnknownEffect(fieldsCodec(table), {
     onExcessProperty: "error",
-  })(value).pipe(Effect.mapError((cause) => schemaMismatch(tableName, id, operation, cause)));
+  })(normalized).pipe(Effect.mapError((cause) => schemaMismatch(tableName, id, operation, cause)));
 });
 
 const decodeFields = Effect.fn("LocalDatabase.decodeFields")(function* (
@@ -492,6 +501,70 @@ export class LocalDatabase extends Context.Service<LocalDatabase, LocalDatabaseS
         }
       });
 
+      const prepareSchema = Effect.fn("LocalDatabase.prepareSchema")(function* (
+        schema: RuntimeSchemaDefinition,
+      ) {
+        const definitions = schema[SchemaDefinitionTypeId];
+        const snapshot = encodeCanonical(SchemaSnapshot, {
+          formatVersion: 2,
+          tables: Object.entries(definitions)
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([name, definition]) => ({
+              name,
+              fields: descriptorFields(definition.fields),
+              indexes: Object.values(definition.indexes).sort((left, right) =>
+                left.name.localeCompare(right.name),
+              ),
+            })),
+        });
+        const current = yield* sql<{
+          snapshot: string;
+        }>`SELECT snapshot FROM schema_state WHERE singleton = 1`;
+        if (current[0]?.snapshot === snapshot) return;
+        let changedData = false;
+        const storedTables = yield* sql<{ id: string; name: string }>`SELECT id, name FROM tables`;
+        for (const stored of storedTables) {
+          if (Object.hasOwn(definitions, stored.name)) continue;
+          const documents =
+            yield* sql`SELECT 1 FROM documents WHERE tableId = ${stored.id} LIMIT 1`;
+          if (documents.length > 0)
+            return yield* Effect.die(`Cannot remove non-empty table '${stored.name}'.`);
+          yield* sql`DELETE FROM tables WHERE id = ${stored.id}`;
+        }
+        for (const [name, definition] of Object.entries(definitions)) {
+          const table = yield* resolveTable(schema, name);
+          let cursor = "";
+          let changed = false;
+          while (true) {
+            const rows = yield* sql<{ id: string; fields: string }>`
+              SELECT id, fields FROM documents WHERE tableId = ${table.id} AND id > ${cursor}
+              ORDER BY id LIMIT 100
+            `;
+            if (rows.length === 0) break;
+            for (const row of rows) {
+              const fields = yield* Effect.try({
+                try: () =>
+                  JSON.stringify(evolveStoredFields(definition.schema, JSON.parse(row.fields))),
+                catch: () =>
+                  new Error(
+                    `Document '${row.id}' in '${name}' does not match the new schema or its conversion rules.`,
+                  ),
+              }).pipe(Effect.orDie);
+              if (fields !== row.fields) {
+                yield* sql`UPDATE documents SET fields = ${fields} WHERE tableId = ${table.id} AND id = ${row.id}`;
+                changed = true;
+                changedData = true;
+              }
+              cursor = row.id;
+            }
+          }
+          if (changed) yield* sql`DELETE FROM indexes WHERE tableId = ${table.id}`;
+        }
+        yield* sql`INSERT INTO schema_state (singleton, snapshot) VALUES (1, ${snapshot})
+          ON CONFLICT (singleton) DO UPDATE SET snapshot = excluded.snapshot`;
+        if (current.length > 0 || changedData) yield* advanceRevision(undefined);
+      });
+
       const indexKeys = Effect.fn("LocalDatabase.indexKeys")(function* (
         table: ResolvedTable,
         row: StoredDocument,
@@ -1026,6 +1099,7 @@ export class LocalDatabase extends Context.Service<LocalDatabase, LocalDatabaseS
       ) =>
         sql.withTransaction(
           Effect.gen(function* () {
+            yield* prepareSchema(schema).pipe(Effect.orDie);
             yield* prepareIndexes(schema).pipe(Effect.orDie);
             const dependencies = makeDependencyRecorder<ReadDependency>();
             const context = Object.freeze({
@@ -1044,6 +1118,7 @@ export class LocalDatabase extends Context.Service<LocalDatabase, LocalDatabaseS
       ) =>
         sql.withTransaction(
           Effect.gen(function* () {
+            yield* prepareSchema(schema).pipe(Effect.orDie);
             yield* prepareIndexes(schema).pipe(Effect.orDie);
             const invalidations = makeDependencyRecorder<WriteInvalidation>();
             const context = Object.freeze({

package/src/dev-runtime/functions.ts

@@ -68,7 +68,7 @@ export type RuntimeHandlerContext = (LocalMutationContext | LocalQueryContext) &
 export type RuntimeFunctionDefinition = InternalRuntimeFunctionDefinition<RuntimeHandlerContext>;
 
 const loadDevelopmentEnvironment = Effect.fn("DevelopmentEnvironment.load")(function* (
-  server: ViteDevServer,
+  server: Pick<ViteDevServer, "ssrLoadModule">,
   appDirectory: string,
 ) {
   const fileSystem = yield* FileSystem.FileSystem;
@@ -100,7 +100,7 @@ export class DevelopmentEnvironment extends Context.Service<
     readonly snapshot: Effect.Effect<LoadedEnvironment, unknown>;
   }
 >()("ignotum/dev-runtime/functions/DevelopmentEnvironment") {
-  static layer(server: ViteDevServer, appDirectory: string) {
+  static layer(server: Pick<ViteDevServer, "ssrLoadModule">, appDirectory: string) {
     return Layer.effect(
       DevelopmentEnvironment,
       Effect.gen(function* () {
@@ -151,7 +151,7 @@ export class FunctionRegistry extends Context.Service<
     ) => Effect.Effect<ResolvedFunction, FunctionResolutionError>;
   }
 >()("ignotum/dev-runtime/functions/FunctionRegistry") {
-  static layer(server: ViteDevServer, appDirectory: string) {
+  static layer(server: Pick<ViteDevServer, "ssrLoadModule">, appDirectory: string) {
     return Layer.effect(
       FunctionRegistry,
       Effect.gen(function* () {

package/src/dev-runtime/id.ts

@@ -1,4 +1,14 @@
-import { Context, Effect, FileSystem, Layer, Path, Schema, Semaphore } from "effect";
+import {
+  Context,
+  Effect,
+  FileSystem,
+  Layer,
+  Path,
+  Predicate,
+  Record,
+  Schema,
+  Semaphore,
+} from "effect";
 import {
   DevelopmentUsername,
   developmentSelectionKey,
@@ -31,9 +41,14 @@ export const LocalCompletion = Schema.Struct({
 
 const projection = (user: User, share: ReadonlyArray<string>): User => ({
   id: user.id,
-  name: share.includes("name") ? user.name : undefined,
-  email: share.includes("email") ? user.email : undefined,
-  image: share.includes("image") ? user.image : undefined,
+  ...Record.filter(
+    {
+      name: share.includes("name") ? user.name : undefined,
+      email: share.includes("email") ? user.email : undefined,
+      image: share.includes("image") ? user.image : undefined,
+    },
+    Predicate.isNotUndefined,
+  ),
 });
 
 export class LocalId extends Context.Service<

package/src/dev-runtime/managed-functions.ts

@@ -0,0 +1,255 @@
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { Crypto, Effect, Encoding, FileSystem, Path, Predicate, Schema } from "effect";
+import type { ViteDevServer } from "vite";
+import { parseSync } from "oxc-parser";
+import { prepareManagedExecution, type ManagedExecution } from "@ignotum/runtime/quickjs";
+import { apiFunctionParts } from "@ignotum/contracts/runtime/sync";
+import { getFunctionEnvironment } from "@ignotum/contracts/runtime/schema";
+import * as serverNamespace from "../server/index.js";
+import * as internalServerNamespace from "../internal/server.js";
+import * as apiNamespace from "../internal/api.js";
+import { validateManagedReleaseGraph } from "@ignotum/versions/managed-release";
+import { descriptorCodec } from "@ignotum/runtime/schema";
+import { SchemaSnapshotField } from "@ignotum/contracts/version";
+import { attachValueDescriptor, defineEnv, emptyEnv } from "@ignotum/contracts/schema";
+import type { ManagedServerBuildResult } from "../cli/build/managed-server.js";
+import type { buildManagedReleaseGraph } from "../cli/managed-release.js";
+
+type DevelopmentModule = Awaited<ReturnType<ViteDevServer["ssrLoadModule"]>>;
+const literal = Schema.encodeSync(Schema.fromJsonString(Schema.String));
+
+export class ManagedFunctionLoadFailed extends Schema.TaggedError<ManagedFunctionLoadFailed>()(
+  "ManagedFunctionLoadFailed",
+  { cause: Schema.Defect() },
+) {}
+
+export interface ManagedFunctionBuild extends Pick<
+  ManagedServerBuildResult,
+  "server" | "artifacts" | "bytes"
+> {
+  readonly environment?: {
+    readonly declaration: Pick<ManagedServerBuildResult["environment"]["declaration"], "fields">;
+  };
+}
+
+export interface ManagedFunctionLoader {
+  readonly reload: (build: ManagedFunctionBuild) => Effect.Effect<void, ManagedFunctionLoadFailed>;
+  readonly invalidate: (cause: unknown) => void;
+  readonly ssrLoadModule: (filename: string) => Promise<DevelopmentModule>;
+}
+
+export interface ManagedFunctionSelection {
+  readonly guest: Effect.Success<ReturnType<typeof buildManagedReleaseGraph>>;
+  readonly release: ManagedExecution["release"];
+  readonly requirements: ManagedExecution["requirements"];
+}
+
+/** Verify the deployed graph contract, then run app code with the local executor's SDK bindings. */
+export const createManagedFunctionLoader = Effect.fn("ManagedFunctions.createLoader")(function* (
+  appDirectory: string,
+  selection: ManagedFunctionSelection,
+) {
+  const fs = yield* FileSystem.FileSystem;
+  const crypto = yield* Crypto.Crypto;
+  const path = yield* Path.Path;
+  const directory = yield* fs.makeTempDirectoryScoped({ prefix: "ignotum-managed-functions-" });
+  const modules = new Map<string, Promise<DevelopmentModule>>();
+  let current = new Map<string, ReadonlyArray<{ readonly name: string; readonly url: string }>>();
+  let environment: Promise<DevelopmentModule> = Promise.resolve({ default: emptyEnv });
+  let failure: unknown;
+  const load = (url: string): Promise<DevelopmentModule> => {
+    let loaded = modules.get(url);
+    if (loaded === undefined) {
+      loaded = import(/* @vite-ignore */ url);
+      modules.set(url, loaded);
+    }
+    return loaded;
+  };
+  const namespaces = new Map<string, DevelopmentModule>([
+    ["ignotum/server", serverNamespace],
+    ["ignotum/internal/server", internalServerNamespace],
+    ["ignotum/internal/api", apiNamespace],
+  ]);
+  const guest = yield* validateManagedReleaseGraph(
+    selection.guest.manifestBytes,
+    [...selection.guest.files].map(([path, bytes]) => ({ path, bytes })),
+  );
+  const bridges = new Map<string, string>();
+  for (const [specifier, namespace] of namespaces) {
+    const entry = guest.manifest.modules.find(
+      (module) => module.path === guest.manifest.entries[specifier],
+    );
+    if (entry === undefined || entry.exports.some((name) => !Object.hasOwn(namespace, name))) {
+      throw new Error(`The local SDK cannot supply managed entry ${specifier}.`);
+    }
+    const filename = path.join(directory, `${encodeURIComponent(specifier)}.mjs`);
+    const key = Symbol.for(filename);
+    // A facade captures the existing SDK namespace so schemas and codecs retain their owner.
+    // The handoff exists only while this closed, generated facade is initialized.
+    yield* Effect.acquireUseRelease(
+      Effect.sync(() => {
+        Reflect.set(globalThis, key, namespace);
+      }),
+      () =>
+        Effect.gen(function* () {
+          const source =
+            `const namespace = globalThis[Symbol.for(${literal(filename)})];\n` +
+            entry.exports
+              .map(
+                (name, index) =>
+                  `const binding${index} = namespace[${literal(name)}]; export { binding${index} as ${name} };`,
+              )
+              .join("\n");
+          yield* fs.writeFileString(filename, source);
+          const url = pathToFileURL(filename).href;
+          yield* Effect.tryPromise({
+            try: () => load(url),
+            catch: (cause) => ManagedFunctionLoadFailed.make({ cause }),
+          });
+          bridges.set(specifier, url);
+        }),
+      () =>
+        Effect.sync(() => {
+          Reflect.deleteProperty(globalThis, key);
+        }),
+    );
+  }
+  const reload = Effect.fn("ManagedFunctions.reload")(
+    function* (build: ManagedFunctionBuild) {
+      const prepared = yield* prepareManagedExecution(
+        {
+          graph: build.server.graph,
+          artifacts: build.artifacts,
+          bytes: build.bytes,
+          entry: build.server.schema.entry,
+          release: selection.release,
+          requirements: selection.requirements,
+          guest: {
+            manifestBytes: selection.guest.manifestBytes,
+            files: [...selection.guest.files].map(([path, bytes]) => ({ path, bytes })),
+          },
+        },
+        "schema",
+      ).pipe(Effect.provideService(Crypto.Crypto, crypto));
+      const urls = new Map(
+        prepared.modules
+          .filter((module) => module.owner === "App")
+          .map((module) => [
+            module.id,
+            pathToFileURL(path.join(directory, `${encodeURIComponent(module.id)}.mjs`)).href,
+          ]),
+      );
+      for (const module of prepared.modules.filter((module) => module.owner === "App")) {
+        const source = prepared.load(module.id);
+        const parsed = parseSync(`${module.id}.mjs`, source, { sourceType: "module" });
+        if (parsed.module.dynamicImports.length > 0)
+          throw new Error("Managed local modules cannot use dynamic imports.");
+        if (parsed.errors.length > 0)
+          throw new Error(`Could not parse managed module ${module.id}.`);
+        const replacements: Array<{ start: number; end: number; value: string }> = [];
+        for (const statement of parsed.program.body) {
+          if (
+            statement.type !== "ImportDeclaration" &&
+            statement.type !== "ExportNamedDeclaration" &&
+            statement.type !== "ExportAllDeclaration"
+          )
+            continue;
+          if (statement.source === null) continue;
+          const targetId = prepared.normalize(module.id, statement.source.value);
+          const target = bridges.get(statement.source.value) ?? urls.get(targetId);
+          if (target === undefined) throw new Error("Missing verified managed module target.");
+          replacements.push({
+            start: statement.source.start,
+            end: statement.source.end,
+            value: literal(target),
+          });
+        }
+        let rewritten = source;
+        for (const replacement of replacements.sort((a, b) => b.start - a.start)) {
+          rewritten =
+            rewritten.slice(0, replacement.start) +
+            replacement.value +
+            rewritten.slice(replacement.end);
+        }
+        if (module.sourceMap !== undefined) {
+          rewritten = rewritten.replace(/^\/\/# sourceMappingURL=.*$/gm, "");
+          rewritten += `\n//# sourceMappingURL=data:application/json;base64,${Encoding.encodeBase64(module.sourceMap.bytes)}\n`;
+        }
+        yield* fs.writeFileString(fileURLToPath(urls.get(module.id)!), rewritten);
+      }
+      const next = new Map<string, Array<{ name: string; url: string }>>();
+      next.set(path.resolve(appDirectory, "server/schema.ts"), [
+        { name: "default", url: urls.get(build.server.schema.entry.module)! },
+      ]);
+      for (const fn of build.server.functions) {
+        const { moduleName, functionName } = apiFunctionParts(fn.address);
+        const filename = path.resolve(appDirectory, "server", `${moduleName}.ts`);
+        const entries = next.get(filename) ?? [];
+        entries.push({ name: functionName, url: urls.get(fn.entry.module)! });
+        next.set(filename, entries);
+      }
+      const first = build.server.functions[0];
+      const fields = yield* Schema.decodeEffect(Schema.Array(SchemaSnapshotField))(
+        build.environment?.declaration.fields ?? [],
+      );
+      const declaredEnvironment =
+        fields.length === 0
+          ? emptyEnv
+          : defineEnv(() =>
+              Object.fromEntries(
+                fields.map((field) => [
+                  field.name,
+                  attachValueDescriptor(
+                    Schema.make<Schema.Codec<unknown>>(descriptorCodec(field.value).ast),
+                    field.value,
+                  ),
+                ]),
+              ),
+            );
+
+      environment =
+        first === undefined
+          ? Promise.resolve({ default: declaredEnvironment })
+          : load(urls.get(first.entry.module)!).then((loaded) => {
+              const candidate = loaded[first.entry.exportName];
+              return {
+                default: Predicate.isObject(candidate)
+                  ? getFunctionEnvironment(candidate)
+                  : undefined,
+              };
+            });
+      yield* Effect.tryPromise({
+        try: () => environment,
+        catch: (cause) => ManagedFunctionLoadFailed.make({ cause }),
+      });
+      current = next;
+      failure = undefined;
+    },
+    Effect.tapCause((cause) =>
+      Effect.sync(() => {
+        failure = cause;
+      }),
+    ),
+    Effect.mapError((cause) => ManagedFunctionLoadFailed.make({ cause })),
+  );
+  const loader: ManagedFunctionLoader = {
+    reload,
+    invalidate: (cause: unknown): void => {
+      failure = cause ?? new Error("Managed server compilation failed.");
+    },
+    ssrLoadModule: async (filename: string): Promise<DevelopmentModule> => {
+      if (failure !== undefined) throw failure;
+      const normalized = path.resolve(filename);
+      if (normalized === path.resolve(appDirectory, "server/env.ts")) return environment;
+      const entries = current.get(normalized);
+      if (entries === undefined)
+        throw new Error(`Module is outside the managed server graph: ${filename}`);
+      return Object.fromEntries(
+        await Promise.all(
+          entries.map(async (entry) => [entry.name, (await load(entry.url)).default]),
+        ),
+      );
+    },
+  };
+  return loader;
+});