@liquidmetal-ai/drizzle 0.0.1

package/src/appify/validate.ts

@@ -0,0 +1,435 @@
+import {
+  Actor,
+  Application,
+  Binding,
+  Bucket,
+  ConfigObject,
+  Domain,
+  Env,
+  Observer,
+  Queue,
+  Service,
+  SqlDatabase,
+  valueOf,
+  VectorIndex,
+  VISIBILITIES,
+} from './build.js';
+import { TokenString } from './parse.js';
+
+// This is a basic linter-style validation framework that uses
+// validators with event handlers to validate the configuration. It
+// takes validators which can implement any event handler, and it
+// traverses the entire manifest.
+export type ValidationError = ConfigObject & {
+  message: string;
+  severity: 'info' | 'error' | 'warning';
+};
+
+export type On<T> = (app: Application, obj: T) => Promise<ValidationError[]>;
+
+export type Validator = {
+  onApplication?: On<Application>;
+  onService?: On<Service>;
+  onActor?: On<Actor>;
+  onObserver?: On<Observer>;
+  onBucket?: On<Bucket>;
+  onQueue?: On<Queue>;
+  onEnv?: On<Env>;
+  onBinding?: On<Binding>;
+  onDomain?: On<Domain>;
+  onSqlDatabase?: On<SqlDatabase>;
+  onVectorIndex?: On<VectorIndex>;
+};
+
+async function validateHelper<T>(
+  validatorFn: On<T> | undefined,
+  application: Application,
+  obj: T,
+  errors: ValidationError[],
+): Promise<void> {
+  if (validatorFn) {
+    const returnedErrors = await validatorFn(application, obj);
+    errors.push(...returnedErrors);
+  }
+}
+
+export async function validate(apps: Application[], validators: Validator[]): Promise<ValidationError[]> {
+  const errors: ValidationError[] = [];
+  for (const app of apps) {
+    for (const validator of validators) {
+      await validateHelper(validator.onApplication, app, app, errors);
+      await validateServices(app, validator, errors);
+      await validateActors(app, validator, errors);
+      await validateObservers(app, validator, errors);
+      await validateBuckets(app, validator, errors);
+      await validateQueues(app, validator, errors);
+      await validateEnvs(app, app, validator, errors);
+      await validateVectorIndexes(app, validator, errors);
+      await validateSqlDatabases(app, validator, errors);
+    }
+  }
+  return errors;
+}
+
+async function validateServices(app: Application, validator: Validator, errors: ValidationError[]): Promise<void> {
+  for (const service of app.service) {
+    await validateHelper(validator.onService, app, service, errors);
+    await validateBindings(app, service, validator, errors);
+    await validateDomains(app, service, validator, errors);
+    await validateEnvs(app, service, validator, errors);
+  }
+}
+
+async function validateActors(app: Application, validator: Validator, errors: ValidationError[]): Promise<void> {
+  for (const actor of app.actor) {
+    await validateHelper(validator.onActor, app, actor, errors);
+    await validateBindings(app, actor, validator, errors);
+    await validateEnvs(app, actor, validator, errors);
+  }
+}
+
+async function validateObservers(app: Application, validator: Validator, errors: ValidationError[]): Promise<void> {
+  for (const observer of app.observer) {
+    await validateHelper(validator.onObserver, app, observer, errors);
+    await validateBindings(app, observer, validator, errors);
+    await validateEnvs(app, observer, validator, errors);
+  }
+}
+
+async function validateBuckets(app: Application, validator: Validator, errors: ValidationError[]): Promise<void> {
+  for (const bucket of app.bucket) {
+    await validateHelper(validator.onBucket, app, bucket, errors);
+  }
+}
+
+async function validateQueues(app: Application, validator: Validator, errors: ValidationError[]): Promise<void> {
+  for (const queue of app.queue) {
+    await validateHelper(validator.onQueue, app, queue, errors);
+  }
+}
+
+async function validateVectorIndexes(app: Application, validator: Validator, errors: ValidationError[]): Promise<void> {
+  for (const vectorIndex of app.vectorIndex) {
+    await validateHelper(validator.onVectorIndex, app, vectorIndex, errors);
+  }
+}
+
+async function validateSqlDatabases(app: Application, validator: Validator, errors: ValidationError[]): Promise<void> {
+  for (const sqlDatabase of app.sqlDatabase) {
+    await validateHelper(validator.onSqlDatabase, app, sqlDatabase, errors);
+  }
+}
+
+async function validateEnvs(
+  app: Application,
+  obj: { env: Env[] },
+  validator: Validator,
+  errors: ValidationError[],
+): Promise<void> {
+  for (const env of obj.env) {
+    await validateHelper(validator.onEnv, app, env, errors);
+  }
+}
+
+async function validateBindings(
+  app: Application,
+  obj: { bindings: Binding[] },
+  validator: Validator,
+  errors: ValidationError[],
+): Promise<void> {
+  for (const binding of obj.bindings) {
+    await validateHelper(validator.onBinding, app, binding, errors);
+  }
+}
+
+async function validateDomains(
+  app: Application,
+  obj: { domains: Domain[] },
+  validator: Validator,
+  errors: ValidationError[],
+): Promise<void> {
+  for (const domain of obj.domains) {
+    await validateHelper(validator.onDomain, app, domain, errors);
+  }
+}
+
+// Validators
+
+const bindingNameValidator: Validator = {
+  onBinding: async (app: Application, binding: Binding): Promise<ValidationError[]> => {
+    const errors: ValidationError[] = [];
+    for (const bind of binding.bindings) {
+      if (!bind.key.value.match(/^[A-Z][A-Z_]*$/)) {
+        errors.push({
+          message: 'binding names should be UPPER_SNAKE_CASE',
+          severity: 'warning',
+          ...bind.key,
+        });
+      }
+    }
+    return errors;
+  },
+};
+
+const bindingValueValidator: Validator = {
+  onBinding: async (app: Application, binding: Binding): Promise<ValidationError[]> => {
+    const errors: ValidationError[] = [];
+    for (const bind of binding.bindings) {
+      if (bind.value.type !== 'string') {
+        errors.push({
+          message: 'bindings must be to names of modules',
+          severity: 'error',
+          ...bind.value,
+        });
+      } else {
+        const ref = valueOf(bind.value);
+        // Walk the app to see if this is a module.
+        let all: { name: TokenString }[] = [];
+        all = all.concat(app.bucket, app.env, app.observer, app.queue, app.service, app.sqlDatabase, app.vectorIndex);
+        if (!all.some((b) => valueOf(b.name) === ref)) {
+          errors.push({
+            message: `binding ${bind.value.value} not found`,
+            severity: 'error',
+            ...bind.value,
+          });
+        } else {
+          // TODO [ian] It _is_ found, so let's make sure it's not
+          // stomping on an implicit binding and check visibility
+          // rules.
+        }
+      }
+    }
+    return errors;
+  },
+};
+
+const domainValidator: Validator = {
+  onDomain: async (app: Application, domain: Domain): Promise<ValidationError[]> => {
+    const errors: ValidationError[] = [];
+    if (domain.fqdn === undefined) {
+      errors.push({
+        message: 'domain must have an fqdn',
+        severity: 'error',
+        ...domain.obj,
+      });
+    } else if (!/^[a-z0-9]+([-.]{1}[a-z0-9]+)*\.[a-z]{2,6}$/.test(valueOf(domain.fqdn))) {
+      errors.push({
+        message: `domain ${domain.fqdn.value} is an invalid domain name`,
+        severity: 'warning',
+        ...domain.fqdn,
+      });
+    }
+    return errors;
+  },
+};
+
+const envValidator: Validator = {
+  onEnv: async (app: Application, env: Env): Promise<ValidationError[]> => {
+    const errors: ValidationError[] = [];
+    if (!valueOf(env.name).match(/^[A-Z][A-Z_]*$/)) {
+      errors.push({
+        message: 'env names should be UPPER_SNAKE_CASE',
+        severity: 'warning',
+        ...env.name,
+      });
+    }
+    if (env.secret?.value === 'true' && env.default !== undefined) {
+      errors.push({
+        message: 'secret env vars cannot have defaults',
+        severity: 'error',
+        ...env.default,
+      });
+    }
+    return errors;
+  },
+};
+
+const observerSourceValidator: Validator = {
+  onObserver: async (app: Application, observer: Observer): Promise<ValidationError[]> => {
+    const errors: ValidationError[] = [];
+    const seen = new Set<string>();
+    for (const source of observer.source) {
+      if (source.bucket && source.queue) {
+        errors.push({
+          message: 'observer source can only have one of bucket or queue',
+          severity: 'error',
+          ...source.obj,
+        });
+      }
+      if (!source.bucket && !source.queue) {
+        errors.push({
+          message: 'observer source must have either a bucket or a queue',
+          severity: 'error',
+          ...source.obj,
+        });
+      }
+      if (
+        source.bucket !== undefined &&
+        !app.bucket.some((b) => valueOf(b.name) === (source.bucket && valueOf(source.bucket)))
+      ) {
+        errors.push({
+          message: `bucket ${source.bucket.value} not found`,
+          severity: 'error',
+          ...source.bucket,
+        });
+      }
+      if (
+        source.queue !== undefined &&
+        !app.queue.some((q) => valueOf(q.name) === (source.queue && valueOf(source.queue)))
+      ) {
+        errors.push({
+          message: `queue ${source.queue.value} not found`,
+          severity: 'error',
+          ...source.queue,
+        });
+      }
+      const sourceName = (source.bucket && valueOf(source.bucket)) || (source.queue && valueOf(source.queue));
+      if (sourceName !== undefined) {
+        if (seen.has(sourceName)) {
+          errors.push({
+            message: `source ${sourceName} is duplicated`,
+            severity: 'error',
+            ...source.obj,
+          });
+        }
+        seen.add(sourceName);
+      }
+      if (source.rule !== undefined && source.queue !== undefined) {
+        errors.push({
+          message: 'queue sources do not have bucket notification rules',
+          severity: 'error',
+          ...source.obj,
+        });
+      }
+    }
+    return errors;
+  },
+};
+
+async function visibilityValidatorFn(app: Application, obj: { visibility?: TokenString }): Promise<ValidationError[]> {
+  const errors: ValidationError[] = [];
+  if (obj.visibility && !(VISIBILITIES as readonly string[]).includes(valueOf(obj.visibility))) {
+    errors.push({
+      message: `visibility must be one of ${VISIBILITIES.join(', ')}`,
+      severity: 'error',
+      ...(obj.visibility || obj),
+    });
+  }
+  return errors;
+}
+
+const visibilityValidator: Validator = {
+  onService: visibilityValidatorFn,
+  onActor: visibilityValidatorFn,
+  onQueue: visibilityValidatorFn,
+  onBucket: visibilityValidatorFn,
+};
+
+async function nameValidatorFn(app: Application, obj: { name: TokenString }): Promise<ValidationError[]> {
+  const errors: ValidationError[] = [];
+  if (!valueOf(obj.name).match(/^[a-z][a-z0-9-]*$/)) {
+    errors.push({
+      message: `name must be lowercase and dash-separated`,
+      severity: 'error',
+      ...obj.name,
+    });
+  }
+  return errors;
+}
+
+const nameValidator: Validator = {
+  onApplication: nameValidatorFn,
+  onService: nameValidatorFn,
+  onObserver: nameValidatorFn,
+  onBucket: nameValidatorFn,
+  onQueue: nameValidatorFn,
+  onActor: nameValidatorFn,
+  onSqlDatabase: nameValidatorFn,
+  onVectorIndex: nameValidatorFn,
+};
+
+const vectorIndexValidator: Validator = {
+  onVectorIndex: async (app: Application, vectorIndex: VectorIndex): Promise<ValidationError[]> => {
+    const errors: ValidationError[] = [];
+    if (vectorIndex.dimension === undefined) {
+      errors.push({
+        message: 'vector index must have a dimension',
+        severity: 'error',
+        ...vectorIndex.obj,
+      });
+    }
+    if (
+      vectorIndex.metric === undefined ||
+      (valueOf(vectorIndex.metric) !== 'cosine' && valueOf(vectorIndex.metric) !== 'euclidean')
+    ) {
+      errors.push({
+        message: 'vector index metric must be either cosine or euclidean',
+        severity: 'error',
+        ...(vectorIndex.metric || vectorIndex.obj),
+      });
+    }
+    if (errors.length > 0) {
+      return errors;
+    }
+    // We have to re-check because the above is too complex for TS.
+    if (vectorIndex.dimension === undefined) {
+      return errors;
+    }
+    const dimension = valueOf(vectorIndex.dimension);
+    if (!Number.isInteger(dimension)) {
+      errors.push({
+        message: 'vector index dimension must be an integer',
+        severity: 'error',
+        ...vectorIndex.dimension,
+      });
+      return errors;
+    }
+    if (dimension < 1 || dimension > 1536) {
+      errors.push({
+        message: 'vector index dimension must be between 1 and 1536',
+        severity: 'error',
+        ...vectorIndex.dimension,
+      });
+    }
+    return errors;
+  },
+};
+
+const duplicateModuleValidator: Validator = {
+  onApplication: async (app: Application, _application: Application): Promise<ValidationError[]> => {
+    const errors: ValidationError[] = [];
+    const seen = new Set<string>();
+    for (const module of [
+      ...app.bucket,
+      ...app.env,
+      ...app.observer,
+      ...app.queue,
+      ...app.service,
+      ...app.sqlDatabase,
+      ...app.vectorIndex,
+    ]) {
+      const name = valueOf(module.name);
+      if (seen.has(name)) {
+        errors.push({
+          message: `module ${name} has already been defined`,
+          severity: 'error',
+          ...module.name,
+        });
+      }
+      seen.add(name);
+    }
+    return errors;
+  },
+};
+
+export const VALIDATORS: Validator[] = [
+  bindingNameValidator,
+  bindingValueValidator,
+  domainValidator,
+  envValidator,
+  observerSourceValidator,
+  visibilityValidator,
+  nameValidator,
+  vectorIndexValidator,
+  duplicateModuleValidator,
+];

package/src/codestore.test.ts

@@ -0,0 +1,98 @@
+import { expect, test } from 'vitest';
+import { Bundle, BundleBase, archive, unarchive } from './codestore.js';
+
+class MemoryBundle extends BundleBase implements Bundle {
+  private files: Map<string, Buffer> = new Map();
+
+  async list(): Promise<string[]> {
+    return Array.from(this.files.keys());
+  }
+
+  async read(name: string): Promise<Buffer> {
+    const buff = this.files.get(name);
+    if (buff === undefined) {
+      throw new Error(`File not found: ${name}`);
+    }
+    return buff;
+  }
+
+  async write(name: string, content: Buffer): Promise<void> {
+    this.files.set(name, content);
+  }
+
+  async delete(name: string): Promise<void> {
+    this.files.delete(name);
+  }
+}
+
+test('archive and unarchive', async () => {
+  const bundle = new MemoryBundle();
+  await bundle.write('file1.txt', Buffer.from('hello'));
+  await bundle.write('file2.txt', Buffer.from('world'));
+  const hash1 = await bundle.hash();
+  const archiveBuffer = await archive(bundle);
+  const bundle2 = new MemoryBundle();
+  await unarchive(archiveBuffer, bundle2);
+  const hash2 = await bundle2.hash();
+  expect(hash1).toBe(hash2);
+  expect(await bundle2.list()).toEqual(['file1.txt', 'file2.txt']);
+  expect((await bundle2.read('file1.txt')).toString()).toBe('hello');
+  expect((await bundle2.read('file2.txt')).toString()).toBe('world');
+});
+
+test('list bundle with files', async () => {
+  const bundle = new MemoryBundle();
+  await bundle.write('file1.txt', Buffer.from('hello'));
+  await bundle.write('file2.txt', Buffer.from('world'));
+  expect(await bundle.list()).toEqual(['file1.txt', 'file2.txt']);
+});
+
+test('read file from bundle', async () => {
+  const bundle = new MemoryBundle();
+  await bundle.write('file1.txt', Buffer.from('hello'));
+  expect((await bundle.read('file1.txt')).toString()).toBe('hello');
+});
+
+test('read missing file from bundle', async () => {
+  const bundle = new MemoryBundle();
+  expect(bundle.read('file1.txt')).rejects.toThrowError('File not found: file1.txt');
+});
+
+test('delete file from bundle', async () => {
+  const bundle = new MemoryBundle();
+  await bundle.write('file1.txt', Buffer.from('hello'));
+  await bundle.delete('file1.txt');
+  expect(bundle.read('file1.txt')).rejects.toThrowError('File not found: file1.txt');
+});
+
+test('for await of bundle', async () => {
+  const bundle = new MemoryBundle();
+  await bundle.write('file1.txt', Buffer.from('hello'));
+  await bundle.write('file2.txt', Buffer.from('world'));
+  const files = [];
+  for await (const file of bundle) {
+    files.push(file.name);
+  }
+  expect(files).toEqual(['file1.txt', 'file2.txt']);
+});
+
+test('stat file from bundle', async () => {
+  const bundle = new MemoryBundle();
+  await bundle.write('file1.txt', Buffer.from('hello'));
+  const stat = await bundle.stat('file1.txt');
+  expect(stat.sizeBytes).toBe(5);
+  expect(stat.sha256Hex).toBe('2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824');
+});
+
+test('stat missing file from bundle', async () => {
+  const bundle = new MemoryBundle();
+  expect(bundle.stat('file1.txt')).rejects.toThrowError('File not found: file1.txt');
+});
+
+test('hash bundle', async () => {
+  const bundle = new MemoryBundle();
+  await bundle.write('file1.txt', Buffer.from('hello'));
+  await bundle.write('file2.txt', Buffer.from('world'));
+  const hash = await bundle.hash();
+  expect(hash).toBe('936a185caaa266bb9cbe981e9e05cb78cd732b0b3280eb944412bb6f8f8f07af');
+});

package/src/codestore.ts

@@ -0,0 +1,93 @@
+import crypto from 'crypto';
+import JSZip from 'jszip';
+
+// CodeStore is a mapping of a handler name to a bundle of code.
+export interface CodeStore {
+  [key: string]: Bundle;
+}
+
+export interface Stat {
+  sizeBytes: number;
+  sha256Hex: string;
+}
+
+export interface ReadableBundle {
+  list(): Promise<string[]>;
+  read(name: string): Promise<Buffer>;
+  stat(name: string): Promise<Stat>;
+  [Symbol.asyncIterator](): AsyncGenerator<File>;
+  hash(): Promise<string>;
+}
+
+export interface WritableBundle {
+  write(name: string, content: Buffer): Promise<void>;
+  delete(name: string): Promise<void>;
+}
+
+export type Bundle = ReadableBundle & WritableBundle;
+
+export interface File {
+  name: string;
+  read(): Promise<Buffer>;
+  stat(): Promise<Stat>;
+}
+
+export abstract class BundleBase implements ReadableBundle {
+  abstract list(): Promise<string[]>;
+  abstract read(name: string): Promise<Buffer>;
+
+  async stat(name: string): Promise<Stat> {
+    const content = await this.read(name);
+    if (content === undefined) {
+      throw new Error(`File not found: ${name}`);
+    }
+    const hash = crypto.createHash('sha256');
+    hash.update(content);
+    const sha256Hex = hash.digest('hex');
+    return {
+      sizeBytes: content.length,
+      sha256Hex,
+    };
+  }
+
+  // This is an AsyncInterator generator function that returns "File"
+  // objects that can be read or stat'd. Use via:
+  // for await (const file of bundle) { ... }
+  async *[Symbol.asyncIterator](): AsyncGenerator<File> {
+    const list = await this.list();
+    for (const name of list) {
+      yield {
+        name,
+        read: async () => await this.read(name),
+        stat: async () => await this.stat(name),
+      };
+    }
+  }
+
+  async hash(): Promise<string> {
+    const hash = crypto.createHash('sha256', { encoding: 'hex' });
+    for await (const file of this) {
+      hash.update(await file.read());
+    }
+    return hash.digest('hex');
+  }
+}
+
+export async function archive(bundle: ReadableBundle): Promise<ArrayBuffer> {
+  const zip = new JSZip();
+  for await (const file of bundle) {
+    zip.file(file.name, await file.read());
+  }
+  return await zip.generateAsync({ type: 'arraybuffer' });
+}
+
+export async function unarchive(zipBuffer: ArrayBuffer, bundle: WritableBundle): Promise<void> {
+  const zip = await JSZip.loadAsync(zipBuffer);
+  for (const [, file] of Object.entries(zip.files)) {
+    if (file.dir) {
+      continue;
+    }
+    const content = await file.async('arraybuffer');
+    await bundle.write(file.name, Buffer.from(content));
+  }
+}