@stringsync/splates 0.0.61 → 1.13.3

package/vendor/@stringsync/splates-server/index-5axhdy8c.js

@@ -0,0 +1,692 @@
+import {
+  replace2,
+  keyIn2,
+  limited
+} from "./index-caep3sde.js";
+
+// replica/dir.ts
+import { dirname } from "node:path";
+
+// replica/replica.ts
+import { createHash } from "node:crypto";
+function md5(bytes) {
+  return createHash("md5").update(bytes).digest("hex");
+}
+function byKey(left, right) {
+  return left.key < right.key ? -1 : left.key > right.key ? 1 : 0;
+}
+
+// replica/dir.ts
+class DirReplica2 {
+  dir;
+  fs;
+  name;
+  constructor(dir, fs) {
+    this.dir = dir;
+    this.fs = fs;
+    this.name = dir;
+  }
+  async put(key, bytes) {
+    await replace2(this.fs, this.path(key), bytes);
+  }
+  async get(key) {
+    const path = this.path(key);
+    if (!await this.fs.exists(path)) {
+      return null;
+    }
+    return this.fs.readBytes(path);
+  }
+  async list(key) {
+    const found = [];
+    const walk = async (at) => {
+      const dir = this.path(at);
+      for (const name of await this.fs.readdir(dir).catch(() => [])) {
+        const child = at === "" ? name : `${at}/${name}`;
+        const stat = await this.fs.stat(`${dir}/${name}`).catch(() => null);
+        if (stat?.isDirectory()) {
+          await walk(child);
+        } else if (stat !== null && !name.endsWith(".tmp")) {
+          const bytes = await this.fs.readBytes(`${dir}/${name}`);
+          found.push({ key: child, size: bytes.length, md5: md5(bytes) });
+        }
+      }
+    };
+    await walk(key);
+    return found.sort(byKey);
+  }
+  async delete(key) {
+    const path = this.path(key);
+    await this.fs.rm(path, { force: true });
+    for (let dir = dirname(path);dir.startsWith(`${this.dir}/`); dir = dirname(dir)) {
+      if ((await this.fs.readdir(dir).catch(() => [null])).length > 0) {
+        break;
+      }
+      await this.fs.rm(dir, { recursive: true, force: true });
+    }
+  }
+  path(key) {
+    return keyIn2(this.dir, key);
+  }
+}
+// replica/place.ts
+import { isAbsolute, resolve } from "node:path";
+
+// replica/aws-sign.ts
+import { createHash as createHash2, createHmac } from "node:crypto";
+
+class AwsSigner {
+  credentials;
+  region;
+  service;
+  constructor(credentials, region, service) {
+    this.credentials = credentials;
+    this.region = region;
+    this.service = service;
+  }
+  sign(request, at) {
+    const stamp = at.toISOString().replace(/[-:]/g, "").replace(/\.\d+/, "");
+    const day = stamp.slice(0, 8);
+    const headers = {
+      ...Object.fromEntries(Object.entries(request.headers).map(([name, value]) => [
+        name.toLowerCase(),
+        value
+      ])),
+      host: request.url.host,
+      "x-amz-date": stamp
+    };
+    if (this.credentials.sessionToken !== undefined) {
+      headers["x-amz-security-token"] = this.credentials.sessionToken;
+    }
+    const names = Object.keys(headers).sort();
+    const canonical = [
+      request.method,
+      request.url.pathname,
+      canonicalQuery(request.url),
+      names.map((name) => `${name}:${headers[name]?.trim()}
+`).join(""),
+      names.join(";"),
+      headers["x-amz-content-sha256"] ?? sha256(request.body)
+    ].join(`
+`);
+    const scope = `${day}/${this.region}/${this.service}/aws4_request`;
+    const toSign = ["AWS4-HMAC-SHA256", stamp, scope, sha256(canonical)].join(`
+`);
+    let key = `AWS4${this.credentials.secretAccessKey}`;
+    for (const part of [day, this.region, this.service, "aws4_request"]) {
+      key = createHmac("sha256", key).update(part).digest();
+    }
+    const signature = createHmac("sha256", key).update(toSign).digest("hex");
+    return {
+      ...headers,
+      Authorization: `AWS4-HMAC-SHA256 Credential=${this.credentials.accessKeyId}/${scope}, SignedHeaders=${names.join(";")}, Signature=${signature}`
+    };
+  }
+}
+function sha256(body) {
+  return createHash2("sha256").update(body).digest("hex");
+}
+function canonicalQuery(url) {
+  return [...url.searchParams.entries()].map(([name, value]) => [encode(name), encode(value)]).sort(([ak, av], [bk, bv]) => ak === bk ? av < bv ? -1 : 1 : ak < bk ? -1 : 1).map(([name, value]) => `${name}=${value}`).join("&");
+}
+function encode(text) {
+  return encodeURIComponent(text).replace(/[!'()*]/g, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`);
+}
+
+// replica/s3.ts
+class S3Replica2 {
+  opts;
+  name;
+  fetch;
+  now;
+  prefix;
+  constructor(opts) {
+    this.opts = opts;
+    this.fetch = opts.fetch ?? fetch;
+    this.now = opts.now ?? (() => new Date);
+    this.prefix = opts.prefix?.replace(/^\/+|\/+$/g, "") ?? "";
+    this.name = `s3://${opts.bucket}${this.prefix === "" ? "" : `/${this.prefix}`}`;
+  }
+  async put(key, bytes) {
+    await this.send("PUT", this.url(key), bytes);
+  }
+  async get(key) {
+    const response = await this.send("GET", this.url(key), "", [404]);
+    return response.status === 404 ? null : new Uint8Array(await response.arrayBuffer());
+  }
+  async list(key) {
+    const under = this.full(key);
+    const found = [];
+    let token = null;
+    do {
+      const url = this.at("");
+      url.searchParams.set("list-type", "2");
+      if (under !== "") {
+        url.searchParams.set("prefix", `${under}/`);
+      }
+      if (token !== null) {
+        url.searchParams.set("continuation-token", token);
+      }
+      const page = parseList(await (await this.send("GET", url, "")).text());
+      for (const object of page.objects) {
+        found.push({ ...object, key: this.relative(object.key) });
+      }
+      token = page.next;
+    } while (token !== null);
+    return found.sort(byKey);
+  }
+  async delete(key) {
+    await this.send("DELETE", this.url(key), "", [404]);
+  }
+  full(key) {
+    return keyIn2(this.prefix, key);
+  }
+  relative(key) {
+    return this.prefix === "" ? key : key.slice(this.prefix.length + 1);
+  }
+  url(key) {
+    return this.at(this.full(key));
+  }
+  at(key) {
+    const path = [this.opts.bucket, ...key.split("/")].filter((part) => part !== "").map(encode).join("/");
+    return new URL(`${this.opts.endpoint.replace(/\/+$/, "")}/${path}`);
+  }
+  async send(method, url, body, allowed = []) {
+    const signer = new AwsSigner(await this.opts.credentials(), this.opts.region ?? "us-east-1", "s3");
+    const headers = signer.sign({
+      method,
+      url,
+      headers: { "x-amz-content-sha256": sha256(body) },
+      body
+    }, this.now());
+    const response = await this.fetch(url, {
+      method,
+      headers,
+      body: method === "PUT" ? new Uint8Array(encodeBody(body)) : undefined
+    });
+    if (!response.ok && !allowed.includes(response.status)) {
+      const text = await response.text().catch(() => "");
+      throw new Error(`${this.name}: ${method} ${url.pathname} answered ${response.status}${errorCode(text)}`);
+    }
+    return response;
+  }
+}
+function parseList(xml) {
+  const objects = [...xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)].map(([, body]) => ({
+    key: unescapeXml(tag(body ?? "", "Key")),
+    size: Number(tag(body ?? "", "Size")),
+    md5: unescapeXml(tag(body ?? "", "ETag")).replace(/"/g, "")
+  }));
+  const truncated = tag(xml, "IsTruncated") === "true";
+  const next = tag(xml, "NextContinuationToken");
+  return { objects, next: truncated && next !== "" ? unescapeXml(next) : null };
+}
+function tag(xml, name) {
+  return xml.match(new RegExp(`<${name}>([\\s\\S]*?)</${name}>`))?.[1] ?? "";
+}
+function unescapeXml(text) {
+  return text.replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&");
+}
+function errorCode(xml) {
+  const code = tag(xml, "Code");
+  return code === "" ? "" : `: ${code}`;
+}
+function encodeBody(body) {
+  return typeof body === "string" ? new TextEncoder().encode(body) : body;
+}
+
+// replica/place.ts
+var REPLICA_ACCESS_KEY2 = "REPLICA_ACCESS_KEY";
+var REPLICA_SECRET_KEY2 = "REPLICA_SECRET_KEY";
+var ASKED = {
+  purpose: "the access key and secret of the bucket this project's `splates.config.ts` names as its `replica`, which splates copies the project's records into a few seconds behind every change",
+  instructions: "In the store's control panel, make an access key limited to the bucket if it can be, and paste its key id as REPLICA_ACCESS_KEY and its secret as REPLICA_SECRET_KEY."
+};
+function openReplica2(place, {
+  root,
+  fs,
+  creds
+}) {
+  if (place.provider === "dir") {
+    return new DirReplica2(isAbsolute(place.path) ? place.path : resolve(root, place.path), fs);
+  }
+  let asked = false;
+  return new S3Replica2({
+    ...place,
+    credentials: async () => {
+      const accessKeyId = await creds.credentials.get(REPLICA_ACCESS_KEY2);
+      const secretAccessKey = await creds.credentials.get(REPLICA_SECRET_KEY2);
+      if (accessKeyId !== undefined && secretAccessKey !== undefined) {
+        return { accessKeyId, secretAccessKey };
+      }
+      if (!asked) {
+        asked = true;
+        for (const name of [REPLICA_ACCESS_KEY2, REPLICA_SECRET_KEY2]) {
+          await creds.request(name, { scope: "project", task: null, ...ASKED });
+        }
+      }
+      throw new Error(`no ${REPLICA_ACCESS_KEY2} and ${REPLICA_SECRET_KEY2} yet: fill them on the Creds page`);
+    }
+  });
+}
+// replica/replicator.ts
+import { Duration } from "../../core/time/index.js";
+import { z as z2 } from "zod";
+
+// replica/layout.ts
+import { z } from "zod";
+var MANIFEST = "manifest.json";
+function valueKey(key) {
+  return `values/${key}`;
+}
+function chunkKey(key, to, last) {
+  return `logs/${key}/${String(to).padStart(10, "0")}-${last}.jsonl`;
+}
+function lineHash(line) {
+  return sha256(line).slice(0, 12);
+}
+function readListing(keys) {
+  const values = new Map;
+  const logs = new Map;
+  for (const { key, md5 } of keys) {
+    if (key.startsWith("values/")) {
+      values.set(key.slice("values/".length), md5);
+      continue;
+    }
+    const chunk = key.match(/^logs\/(.+)\/(\d{10})-([0-9a-f]{12})\.jsonl$/);
+    if (chunk?.[1] === undefined) {
+      continue;
+    }
+    const chunks = logs.get(chunk[1]) ?? [];
+    chunks.push({ key, to: Number(chunk[2]), last: chunk[3] ?? "" });
+    logs.set(chunk[1], chunks);
+  }
+  for (const chunks of logs.values()) {
+    chunks.sort((left, right) => left.to - right.to);
+  }
+  return { values, logs };
+}
+var MANIFEST_SCHEMA = z.object({
+  version: z.literal(1),
+  origin: z.string(),
+  host: z.string(),
+  caughtUpAt: z.number().nullable()
+});
+async function readManifest(replica) {
+  const bytes = await replica.get(MANIFEST);
+  if (bytes === null) {
+    return null;
+  }
+  const parsed = MANIFEST_SCHEMA.safeParse(JSON.parse(new TextDecoder().decode(bytes)));
+  if (!parsed.success) {
+    throw new Error(`${replica.name} has a ${MANIFEST} splates cannot read`);
+  }
+  return parsed.data;
+}
+
+// replica/shipped.ts
+var KEPT_BACK = /(^|\/)creds\.json$|\.lock(\/|$)|\.tmp$|^replica\.json$/;
+function replicated2(key) {
+  return !KEPT_BACK.test(key);
+}
+
+// replica/replicator.ts
+class Replicator2 {
+  opts;
+  batch;
+  maxRetry;
+  values = new Map;
+  logs = new Map;
+  pending = new Set;
+  behindSince = null;
+  caughtUpAt = null;
+  error = null;
+  origin = null;
+  failures = 0;
+  watch = null;
+  scheduled = null;
+  flushing = null;
+  stopped = false;
+  recordedAs = "";
+  claimed = false;
+  constructor(opts) {
+    this.opts = opts;
+    this.batch = opts.batch ?? Duration.secs(3);
+    this.maxRetry = opts.maxRetry ?? Duration.mins(5);
+  }
+  async start() {
+    this.watch = this.opts.storage.watch("", { recursive: true });
+    this.watch.events.on("changed", (key) => this.changed(key));
+    await this.flush();
+  }
+  status() {
+    return {
+      replica: this.opts.replica.name,
+      caughtUpAt: this.caughtUpAt,
+      pending: this.pending.size,
+      behindSince: this.behindSince,
+      error: this.error
+    };
+  }
+  static async recorded(storage) {
+    const record = await storage.document(RECORD_KEY, { schema: RECORD }).read();
+    return record?.status ?? null;
+  }
+  static async adopt(storage, origin) {
+    await storage.document(RECORD_KEY, { schema: RECORD }).update((current) => ({ ...current, origin }));
+  }
+  async flush() {
+    while (this.flushing !== null) {
+      await this.flushing;
+    }
+    this.scheduled?.dispose();
+    this.scheduled = null;
+    this.flushing = this.run().finally(() => {
+      this.flushing = null;
+    });
+    await this.flushing;
+  }
+  async run() {
+    if (this.origin === null && !await this.catchUp()) {
+      await this.record();
+      return;
+    }
+    await this.ship();
+    await this.record();
+  }
+  dispose() {
+    this.stopped = true;
+    this.watch?.dispose();
+    this.scheduled?.dispose();
+  }
+  changed(key) {
+    if (key === CREDS_KEY && this.error !== null && !this.stopped) {
+      this.flush();
+    }
+    if (!replicated2(key)) {
+      return;
+    }
+    this.pending.add(key);
+    this.behindSince ??= this.opts.clock.now();
+    if (this.origin !== null) {
+      this.schedule(this.batch);
+    }
+  }
+  schedule(after) {
+    if (this.scheduled !== null || this.stopped) {
+      return;
+    }
+    this.scheduled = this.opts.timer.setTimeout(() => {
+      this.scheduled = null;
+      this.flush();
+    }, after);
+  }
+  async catchUp() {
+    const { storage, replica } = this.opts;
+    try {
+      const manifest = await this.manifest();
+      const origin = await this.ownOrigin();
+      if (manifest !== null && manifest.origin !== origin) {
+        this.error = `${replica.name} holds the records of another server, last caught up on ${manifest.host}: restore from it, or give this project's replica its own prefix`;
+        this.opts.log.error(this.error);
+        return false;
+      }
+      const listing = readListing(await replica.list(""));
+      this.values = listing.values;
+      this.logs = listing.logs;
+      this.origin = origin;
+      this.caughtUpAt = manifest?.caughtUpAt ?? null;
+      const keys = new Set([
+        ...await storage.keys(""),
+        ...this.values.keys(),
+        ...this.logs.keys()
+      ]);
+      for (const key of keys) {
+        if (replicated2(key)) {
+          this.pending.add(key);
+        }
+      }
+      this.behindSince ??= this.opts.clock.now();
+    } catch (error) {
+      this.failed(error);
+      return false;
+    }
+    return true;
+  }
+  async ship() {
+    const keys = [...this.pending].sort();
+    this.pending.clear();
+    const startedAt = this.opts.clock.now();
+    const failed = [];
+    await limited(keys, CONCURRENCY, async (key) => {
+      try {
+        await this.shipKey(key);
+      } catch (error) {
+        failed.push(error);
+        this.pending.add(key);
+      }
+    });
+    if (failed.length === 0 && (keys.length > 0 || !this.claimed)) {
+      try {
+        await this.writeManifest(this.pending.size === 0 ? startedAt : null);
+      } catch (error) {
+        failed.push(error);
+      }
+    }
+    if (failed.length > 0) {
+      this.failed(failed[0]);
+      return;
+    }
+    this.failures = 0;
+    this.error = null;
+    if (this.pending.size === 0) {
+      this.behindSince = null;
+    } else {
+      this.schedule(this.batch);
+    }
+  }
+  async shipKey(key) {
+    const { storage, replica } = this.opts;
+    const chunks = this.logs.get(key) ?? [];
+    const last = chunks.at(-1);
+    const kept = await storage.copy(key, last === undefined ? 0 : last.to - 1);
+    if (kept?.kind !== "value" && this.values.has(key)) {
+      await replica.delete(valueKey(key));
+      this.values.delete(key);
+    }
+    if (kept?.kind !== "log" && chunks.length > 0) {
+      await this.dropChunks(key);
+    }
+    if (kept?.kind === "value") {
+      const hash = md5(kept.bytes);
+      if (this.values.get(key) !== hash) {
+        await replica.put(valueKey(key), kept.bytes);
+        this.values.set(key, hash);
+      }
+    } else if (kept?.kind === "log") {
+      await this.shipLines(key, kept, last);
+    }
+  }
+  async shipLines(key, kept, last) {
+    let lines = kept.lines;
+    let from = 0;
+    if (last !== undefined) {
+      const [overlap, ...after] = kept.lines;
+      if (kept.count >= last.to && overlap !== undefined && lineHash(overlap) === last.last) {
+        lines = after;
+        from = last.to;
+      } else {
+        await this.dropChunks(key);
+        const again = await this.opts.storage.copy(key);
+        lines = again?.kind === "log" ? again.lines : [];
+      }
+    }
+    const final = lines.at(-1);
+    if (final === undefined) {
+      return;
+    }
+    const chunk = {
+      key: chunkKey(key, from + lines.length, lineHash(final)),
+      to: from + lines.length,
+      last: lineHash(final)
+    };
+    await this.opts.replica.put(chunk.key, new TextEncoder().encode(lines.map((line) => `${line}
+`).join("")));
+    this.logs.set(key, [...this.logs.get(key) ?? [], chunk]);
+  }
+  async dropChunks(key) {
+    for (const chunk of this.logs.get(key) ?? []) {
+      await this.opts.replica.delete(chunk.key);
+    }
+    this.logs.delete(key);
+  }
+  manifest() {
+    return readManifest(this.opts.replica);
+  }
+  async writeManifest(at) {
+    if (at !== null) {
+      this.caughtUpAt = at;
+    }
+    const manifest = {
+      version: 1,
+      origin: this.origin ?? "",
+      host: this.opts.host,
+      caughtUpAt: this.caughtUpAt
+    };
+    await this.opts.replica.put(MANIFEST, new TextEncoder().encode(`${JSON.stringify(manifest, null, "\t")}
+`));
+    this.claimed = true;
+  }
+  async ownOrigin() {
+    const kept = await this.opts.storage.document(RECORD_KEY, { schema: RECORD }).update((current) => current ?? { origin: crypto.randomUUID() });
+    return kept.origin;
+  }
+  async record() {
+    const status = this.status();
+    const text = JSON.stringify(status);
+    if (text === this.recordedAs) {
+      return;
+    }
+    this.recordedAs = text;
+    try {
+      await this.opts.storage.document(RECORD_KEY, { schema: RECORD }).update((current) => ({
+        origin: current?.origin ?? this.origin ?? crypto.randomUUID(),
+        status
+      }));
+    } catch (error) {
+      this.recordedAs = "";
+      this.opts.log.error(`replica status not recorded: ${error}`);
+    }
+  }
+  failed(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (message !== this.error) {
+      this.opts.log.error(`replica ${this.opts.replica.name}: ${message}`);
+    }
+    this.error = message;
+    this.failures++;
+    const wait = Math.min(this.batch.ms * 2 ** this.failures, this.maxRetry.ms);
+    this.schedule(Duration.ms(wait));
+  }
+}
+var RECORD_KEY = "replica.json";
+var CREDS_KEY = "creds.json";
+var STATUS = z2.object({
+  replica: z2.string(),
+  caughtUpAt: z2.number().nullable(),
+  pending: z2.number(),
+  behindSince: z2.number().nullable(),
+  error: z2.string().nullable()
+});
+var RECORD = z2.object({
+  origin: z2.string(),
+  status: STATUS.optional()
+});
+var CONCURRENCY = 8;
+// replica/restore.ts
+async function restoreFromReplica2(replica, {
+  target,
+  fs,
+  format,
+  force
+}) {
+  const manifest = await readManifest(replica);
+  if (manifest === null) {
+    throw new Error(`nothing to restore: ${replica.name} holds no records yet`);
+  }
+  const present = await target.exists();
+  if (present && !force) {
+    throw new Error("this project already has records here: restore over them with --force, which leaves them aside");
+  }
+  const listing = readListing(await replica.list(""));
+  const stamp = listing.values.has("format") ? Number(new TextDecoder().decode(await need(replica, "values/format"))) : format;
+  if (stamp > format) {
+    throw new Error(`the replica's records are in format ${stamp}, newer than this splates's ${format}: update splates first`);
+  }
+  const staging = `${await target.dir()}/restoring.tmp`;
+  await fs.rm(staging, { recursive: true, force: true });
+  await fs.mkdir(staging, { recursive: true });
+  try {
+    let values = 0;
+    let logs = 0;
+    for (const key of listing.values.keys()) {
+      if (!replicated2(key)) {
+        continue;
+      }
+      await replace2(fs, `${staging}/${key}`, await need(replica, `values/${key}`));
+      values++;
+    }
+    for (const [key, chunks] of listing.logs) {
+      if (!replicated2(key)) {
+        continue;
+      }
+      const parts = [];
+      for (const chunk of chunks) {
+        parts.push(await need(replica, chunk.key));
+      }
+      await replace2(fs, `${staging}/${key}`, join(parts));
+      logs++;
+    }
+    const replaced = present ? await target.moveAside(`the replica ${replica.name}, restored here`) : null;
+    await target.import(staging, stamp);
+    await target.clearMoved();
+    await Replicator2.adopt(await target.open(), manifest.origin);
+    return { values, logs, host: manifest.host, replaced };
+  } finally {
+    target.dispose();
+    await fs.rm(staging, { recursive: true, force: true });
+  }
+}
+async function need(replica, key) {
+  const bytes = await replica.get(key);
+  if (bytes === null) {
+    throw new Error(`${replica.name} lists ${key} but no longer holds it`);
+  }
+  return bytes;
+}
+function join(parts) {
+  const whole = new Uint8Array(parts.reduce((sum, part) => sum + part.length, 0));
+  let at = 0;
+  for (const part of parts) {
+    whole.set(part, at);
+    at += part.length;
+  }
+  return whole;
+}
+// replica/status.ts
+import { Duration as Duration2, timeAgo } from "../../core/time/index.js";
+function statusLines2(status, named, now) {
+  if (status === null) {
+    return [
+      named ? "no replica yet: `splates server` starts it as it serves the project" : "no replica: name one as `replica` in splates.config.ts, and `splates server` keeps it"
+    ];
+  }
+  const ago = (at) => timeAgo(Duration2.ms(now - at));
+  const { replica, caughtUpAt, behindSince, pending, error } = status;
+  return [
+    `replica:    ${replica}`,
+    `caught up:  ${caughtUpAt === null ? "never" : `${ago(caughtUpAt)} (${new Date(caughtUpAt).toISOString()})`}`,
+    `behind by:  ${behindSince === null ? "nothing" : `${pending} change${pending === 1 ? "" : "s"}, the oldest ${ago(behindSince)}`}`,
+    ...error === null ? [] : [`failing:    ${error}`]
+  ];
+}
+export { DirReplica2, S3Replica2, REPLICA_ACCESS_KEY2, REPLICA_SECRET_KEY2, openReplica2, replicated2, Replicator2, restoreFromReplica2, statusLines2 };