@terrafirmagreg-team/dump 0.0.0-stage → 1.2.0

package/src/lib/dump/json-reader.ts

@@ -0,0 +1,254 @@
+import { parseDumpManifest } from '@terrafirmagreg-team/dump/lib/dump/manifest/parse';
+import {
+  dumpStorageUrl,
+  registerDumpLocation,
+  type DumpLocation,
+} from '@terrafirmagreg-team/dump/lib/dump/paths';
+import type { DatasetRef } from '@terrafirmagreg-team/sites/location';
+import type { DumpReader } from '@terrafirmagreg-team/dump/lib/dump/reader';
+import { createAsyncCache } from '@terrafirmagreg-team/data/lib/async-cache';
+
+const SAFE_LANG_CODE = /^[a-z0-9]+(_[a-z0-9]+)*$/i;
+
+function cleanLangCode(code: string): string | undefined {
+  return SAFE_LANG_CODE.test(code) ? code : undefined;
+}
+
+const MAX_INFLIGHT = 12;
+const lanes = {
+  high: [] as Array<() => void>,
+  normal: [] as Array<() => void>,
+};
+let inflight = 0;
+
+function pump() {
+  while (inflight < MAX_INFLIGHT) {
+    const run = lanes.high.shift() ?? lanes.normal.shift();
+    if (!run) return;
+    inflight++;
+    run();
+  }
+}
+
+function fetchLane<T>(task: () => Promise<T>, high: boolean): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const run = () =>
+      task()
+        .then(resolve, reject)
+        .finally(() => {
+          inflight--;
+          pump();
+        });
+    (high ? lanes.high : lanes.normal).push(run);
+    pump();
+  });
+}
+
+const FETCH_TIMEOUT = 20_000;
+
+async function request(url: string): Promise<Response | undefined> {
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try {
+      return await fetch(url, {
+        signal: AbortSignal.timeout(FETCH_TIMEOUT),
+      });
+    } catch {
+
+    }
+  }
+  return undefined;
+}
+
+export const DUMP_POINTER_TTL = 60_000;
+
+const fetchJsonLoader = async (url: string, high: boolean) =>
+  fetchLane(async () => {
+    try {
+      const res = await request(url);
+      if (!res?.ok) return undefined;
+      return (await res.json()) as unknown;
+    } catch {
+      return undefined;
+    }
+  }, high);
+
+const fetchJsonGzLoader = async (url: string, high: boolean) =>
+  fetchLane(async () => {
+    try {
+      const res = await request(url);
+      if (!res?.ok || !res.body) return undefined;
+      const text = await new Response(
+        res.body.pipeThrough(new DecompressionStream('gzip')),
+      ).text();
+      return JSON.parse(text) as unknown;
+    } catch {
+      return undefined;
+    }
+  }, high);
+
+const fetchJson = createAsyncCache(fetchJsonLoader, (url) => url);
+
+const fetchJsonGz = createAsyncCache(fetchJsonGzLoader, (url) => url);
+
+const fetchJsonMutable = createAsyncCache(
+  fetchJsonLoader,
+  (url) => url,
+  DUMP_POINTER_TTL,
+);
+
+const fetchJsonGzMutable = createAsyncCache(
+  fetchJsonGzLoader,
+  (url) => url,
+  DUMP_POINTER_TTL,
+);
+
+export const COMPRESSED_TREE = /^(?:data|meta)\//;
+
+function toLangMap(raw: unknown): Record<string, string> {
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {};
+  const map: Record<string, string> = {};
+  for (const [key, value] of Object.entries(raw)) {
+    map[key] = typeof value === 'string' ? value : JSON.stringify(value);
+  }
+  return map;
+}
+
+export class JsonDumpReader implements DumpReader {
+  private manifestCache: Promise<ReturnType<typeof parseDumpManifest>> | undefined;
+  private locationCache: Promise<DumpLocation> | undefined;
+  private langCache = createAsyncCache((code: string) => this.loadLang(code));
+  private langRawCache = createAsyncCache((code: string) =>
+    this.loadLangRaw(code),
+  );
+  private readonly packId: string;
+  private readonly version?: string;
+
+  constructor(private dataset: DatasetRef) {
+    const ref = dataset;
+    this.packId = ref.id;
+    this.version = ref.version;
+  }
+
+  getJson(path: string, priority?: 'high'): Promise<unknown | undefined> {
+    const high = priority === 'high';
+    if (path === 'versions.json') {
+      return fetchJsonMutable(
+        dumpStorageUrl(`${this.packId}/versions.json`),
+        high,
+      );
+    }
+    return this.location().then((loc) => {
+      const key = loc.files?.[path];
+      if (key) {
+        return key.endsWith('.gz')
+          ? fetchJsonGz(dumpStorageUrl(`${key}`), high)
+          : fetchJson(dumpStorageUrl(`${key}`), high);
+      }
+
+      if (loc.files) return undefined;
+      const rel = `${loc.root}/${path}`;
+      const [json, jsonGz] =
+        loc.root === this.packId
+          ? [fetchJsonMutable, fetchJsonGzMutable]
+          : [fetchJson, fetchJsonGz];
+      return COMPRESSED_TREE.test(path)
+        ? jsonGz(dumpStorageUrl(`${rel}.gz`), high)
+        : json(dumpStorageUrl(`${rel}`), high);
+    });
+  }
+
+  getLang(lang: string): Promise<Record<string, string>> {
+    const code = cleanLangCode(lang);
+    return code ? this.langCache(code) : Promise.resolve({});
+  }
+
+  getLangRaw(lang: string): Promise<Record<string, string>> {
+    const code = cleanLangCode(lang);
+    return code ? this.langRawCache(code) : Promise.resolve({});
+  }
+
+  private location(): Promise<DumpLocation> {
+    const p = (this.locationCache ??= this.resolveLocation());
+    p.catch(() => {
+      if (this.locationCache === p) this.locationCache = undefined;
+    });
+    return p;
+  }
+
+  private async resolveLocation(): Promise<DumpLocation> {
+    if (this.version) {
+      const root = `${this.packId}/${this.version}`;
+      const loc: DumpLocation = { root, files: await this.objectsIndex(root) };
+      registerDumpLocation(this.dataset, loc);
+      return loc;
+    }
+    const versions = (await fetchJsonMutable(
+      dumpStorageUrl(`${this.packId}/versions.json`),
+      true,
+    )) as { latest?: string } | undefined;
+    if (versions?.latest) {
+      const root = `${this.packId}/${versions.latest}`;
+
+      const exists = await fetchJson(
+        dumpStorageUrl(`${root}/manifest.json`),
+        false,
+      );
+      if (exists) {
+        const loc: DumpLocation = { root, files: await this.objectsIndex(root) };
+        registerDumpLocation(this.dataset, loc);
+        registerDumpLocation({ id: this.packId, version: versions.latest }, loc);
+        return loc;
+      }
+    }
+    const loc: DumpLocation = { root: this.packId };
+    registerDumpLocation(this.dataset, loc);
+    return loc;
+  }
+
+  private async objectsIndex(
+    root: string,
+  ): Promise<Record<string, string> | undefined> {
+    const raw = await fetchJson(dumpStorageUrl(`${root}/objects.json`), false);
+    const files =
+      raw && typeof raw === 'object' && !Array.isArray(raw)
+        ? (raw as Record<string, unknown>).files
+        : undefined;
+    if (!files || typeof files !== 'object' || Array.isArray(files)) {
+      return undefined;
+    }
+    const out: Record<string, string> = {};
+    for (const [rel, key] of Object.entries(files)) {
+      if (typeof key === 'string') out[rel] = key;
+    }
+    return out;
+  }
+
+  private manifest() {
+    const p = (this.manifestCache ??= this.getJson('manifest.json').then((raw) =>
+      parseDumpManifest(raw),
+    ));
+    p.catch(() => {
+      if (this.manifestCache === p) this.manifestCache = undefined;
+    });
+    return p;
+  }
+
+  private async loadLang(lang: string): Promise<Record<string, string>> {
+    const codes = lang === 'en_us' ? ['en_us'] : ['en_us', lang];
+    const loc = await this.location();
+    const base = loc.files ? 'lang' : 'meta/lang';
+    const metaMaps = await Promise.all(
+      codes.map((code) =>
+        this.getJson(`${base}/${code}.json`).then(toLangMap),
+      ),
+    );
+    return Object.assign({}, ...metaMaps);
+  }
+
+  private async loadLangRaw(code: string): Promise<Record<string, string>> {
+    const loc = await this.location();
+    const base = loc.files ? 'lang' : 'meta/lang';
+    return this.getJson(`${base}/${code}.json`).then(toLangMap);
+  }
+}
+

package/src/lib/dump/manifest/dto.ts

@@ -0,0 +1,12 @@
+export type {
+  Display as NamespaceDisplayText,
+  DumpIndexes,
+  DumpProducer,
+  DumpText,
+  Manifest as DumpManifest,
+  NamespaceDisplay,
+  PackInfo,
+  RecipeTypeRow,
+  RecipeTypesIndex,
+} from '@terrafirmagreg-team/dump/lib/dump/gen';
+

package/src/lib/dump/manifest/parse.ts

@@ -0,0 +1,168 @@
+import {
+  asNumber,
+  asRecord,
+  asString,
+  asStringList,
+} from '@terrafirmagreg-team/dump/lib/dump/contract/read';
+import type {
+  DumpIndexes,
+  DumpManifest,
+  DumpProducer,
+  DumpText,
+  NamespaceDisplay,
+  PackInfo,
+  RecipeTypeRow,
+} from '@terrafirmagreg-team/dump/lib/dump/manifest/dto';
+
+function asProducer(value: unknown): DumpProducer | undefined {
+  const row = asRecord(value);
+  if (!row) return undefined;
+  const name = asString(row.name);
+  const platform = asString(row.platform);
+  const minecraft = asString(row.minecraft);
+  if (!name || !platform || !minecraft) return undefined;
+  return { name, platform, minecraft };
+}
+
+function asPackInfo(value: unknown): PackInfo | undefined {
+  const row = asRecord(value);
+  if (!row) return undefined;
+  const id = asString(row.id);
+  if (!id) return undefined;
+  return { id, version: typeof row.version === 'string' ? row.version : null };
+}
+
+function asDumpText(value: unknown): DumpText | undefined {
+  const row = asRecord(value);
+  if (!row) return undefined;
+  const out: DumpText = {};
+  const text = asString(row.text);
+  const translate = asString(row.translate);
+  const keybind = asString(row.keybind);
+  if (text !== undefined) out.text = text;
+  if (translate !== undefined) out.translate = translate;
+  if (keybind !== undefined) out.keybind = keybind;
+  if (Array.isArray(row.with)) {
+    out.with = row.with.flatMap((entry) => {
+      const part = asDumpText(entry);
+      return part ? [part] : [];
+    });
+  }
+  if (Array.isArray(row.extra)) {
+    out.extra = row.extra.flatMap((entry) => {
+      const part = asDumpText(entry);
+      return part ? [part] : [];
+    });
+  }
+  return out;
+}
+
+function asDisplay(value: unknown): NamespaceDisplay['display'] | undefined {
+  const row = asRecord(value);
+  if (!row) return undefined;
+  const name = asString(row.name);
+  if (!name) return undefined;
+  const tooltipMods: Record<string, DumpText[]> = {};
+  const modsRow = asRecord(row.tooltipMods);
+  if (modsRow) {
+    for (const [mod, entries] of Object.entries(modsRow)) {
+      if (!Array.isArray(entries)) continue;
+      tooltipMods[mod] = entries.flatMap((entry) => {
+        const text = asDumpText(entry);
+        return text ? [text] : [];
+      });
+    }
+  }
+  return {
+    name,
+    tooltip: Array.isArray(row.tooltip)
+      ? row.tooltip.flatMap((entry) => {
+          const text = asDumpText(entry);
+          return text ? [text] : [];
+        })
+      : [],
+    tooltipMods,
+  };
+}
+
+function asNamespaces(
+  value: unknown,
+): Record<string, NamespaceDisplay> | undefined {
+  const row = asRecord(value);
+  if (!row) return undefined;
+  const out: Record<string, NamespaceDisplay> = {};
+  for (const [ns, entry] of Object.entries(row)) {
+    if (!ns || ns.includes('/') || ns.includes('\\') || ns.includes('..')) {
+      continue;
+    }
+    const nsRow = asRecord(entry);
+    const display = asDisplay(nsRow?.display);
+    if (!display) continue;
+    out[ns] = {
+      display,
+      registry: asStringList(nsRow?.registry) ?? [],
+      guides: asStringList(nsRow?.guides) ?? [],
+      languages: asStringList(nsRow?.languages) ?? [],
+      quests: nsRow?.quests === true,
+      keybinds: nsRow?.keybinds === true,
+    };
+  }
+  return out;
+}
+
+function asRecipeTypeRow(value: unknown): RecipeTypeRow | undefined {
+  const row = asRecord(value);
+  if (!row) return undefined;
+  const id = asString(row.id);
+  const count = asNumber(row.count);
+  const filesRow = asRecord(row.files);
+  if (!id || count === undefined || !filesRow) return undefined;
+  const files: Record<string, string> = {};
+  for (const [ns, path] of Object.entries(filesRow)) {
+    if (typeof path === 'string' && path) files[ns] = path;
+  }
+  return { id, count, files };
+}
+
+function asIndexes(value: unknown): DumpIndexes | undefined {
+  const row = asRecord(value);
+  if (!row) return undefined;
+  const recipesRow = asRecord(row.recipes);
+  if (!recipesRow || !Array.isArray(recipesRow.types)) return undefined;
+  const types = recipesRow.types.flatMap((entry) => {
+    const typeRow = asRecipeTypeRow(entry);
+    return typeRow ? [typeRow] : [];
+  });
+  const quests: Record<string, string> = {};
+  const questsRow = asRecord(row.quests);
+  if (questsRow) {
+    for (const [ns, path] of Object.entries(questsRow)) {
+      if (typeof path === 'string' && path) quests[ns] = path;
+    }
+  }
+  return {
+    recipes: { types },
+    quests,
+    used: asStringList(row.used) ?? [],
+  };
+}
+
+export function parseDumpManifest(value: unknown): DumpManifest | undefined {
+  const row = asRecord(value);
+  if (!row) return undefined;
+  if (asString(row.schema) !== 'manifest') return undefined;
+  const producer = asProducer(row.producer);
+  const indexes = asIndexes(row.indexes);
+  const namespaces = asNamespaces(row.namespaces);
+  if (!producer || !indexes || !namespaces) return undefined;
+  return {
+    schema: 'manifest',
+    producer,
+    pack: asPackInfo(row.pack) ?? null,
+    languages: asStringList(row.languages),
+    layout: asString(row.layout) ?? 'v1',
+    namespaces,
+    indexes,
+  };
+}
+

package/src/lib/dump/mods/dto.ts

@@ -0,0 +1,31 @@
+import type { DumpManifest } from '@terrafirmagreg-team/dump/lib/dump/manifest/dto';
+
+export type GuideBookRef = {
+  modId: string;
+  bookPath: string;
+  bookId: string;
+  dir: string;
+};
+
+export type DumpMod = {
+  namespace: string;
+  keybindsFile: string | null;
+  items: string | null;
+  blocks: string | null;
+  fluids: string | null;
+  entities: string | null;
+  tabs: string | null;
+  groups: string | null;
+  recipeCategories: string | null;
+  tags: string | null;
+  recipeIndex: string | null;
+  questData: string | null;
+  books: GuideBookRef[];
+};
+
+export type DumpIndex = {
+  root: string;
+  manifest: DumpManifest;
+  mods: DumpMod[];
+};
+

package/src/lib/dump/mods/index.ts

@@ -0,0 +1,97 @@
+import type {
+  DumpManifest,
+  NamespaceDisplay,
+} from '@terrafirmagreg-team/dump/lib/dump/manifest/dto';
+import type { DumpIndex, DumpMod, GuideBookRef } from '@terrafirmagreg-team/dump/lib/dump/mods/dto';
+
+function registryRel(
+  meta: NamespaceDisplay,
+  file: string,
+): string | null {
+  return meta.registry.includes(file) ? `registry/${file}.json` : null;
+}
+
+function loadBooks(
+  namespace: string,
+  guides: string[],
+): GuideBookRef[] {
+  const books: GuideBookRef[] = [];
+  for (const name of guides) {
+    if (!name || name.includes('..') || name.includes('/')) continue;
+    books.push({
+      modId: namespace,
+      bookPath: name,
+      bookId: `${namespace}:${name}`,
+      dir: `data/${namespace}/guides/${name}`,
+    });
+  }
+  return books;
+}
+
+function loadMod(
+  namespace: string,
+  meta: NamespaceDisplay,
+): DumpMod {
+  return {
+    namespace,
+    keybindsFile: meta.keybinds ? 'keybinds.json' : null,
+    items: registryRel(meta, 'items'),
+    blocks: registryRel(meta, 'blocks'),
+    fluids: registryRel(meta, 'fluids'),
+    entities: registryRel(meta, 'entities'),
+    tabs: registryRel(meta, 'tabs'),
+    groups: registryRel(meta, 'groups'),
+    recipeCategories: registryRel(meta, 'recipe_categories'),
+    tags: registryRel(meta, 'tags'),
+    recipeIndex: registryRel(meta, 'recipes'),
+    questData: meta.quests ? `data/${namespace}/quests/data.json` : null,
+    books: loadBooks(namespace, meta.guides),
+  };
+}
+
+export function dumpIndexFromManifest(
+  manifest: DumpManifest,
+  root: string,
+): DumpIndex {
+  const mods = Object.keys(manifest.namespaces)
+    .sort((a, b) => a.localeCompare(b))
+    .flatMap((namespace) => {
+      const row = manifest.namespaces[namespace];
+      return row ? [loadMod(namespace, row)] : [];
+    });
+  return { root, manifest, mods };
+}
+
+const REGISTRY_FIELD = {
+  items: 'items',
+  blocks: 'blocks',
+  fluids: 'fluids',
+  entities: 'entities',
+  tabs: 'tabs',
+  groups: 'groups',
+  recipe_categories: 'recipeCategories',
+  tags: 'tags',
+  recipes: 'recipeIndex',
+} as const satisfies Record<string, keyof DumpMod>;
+
+export function dumpRegistryRel(
+  dump: DumpIndex | null | undefined,
+  file: string,
+): string | undefined {
+  const field = REGISTRY_FIELD[file as keyof typeof REGISTRY_FIELD];
+  if (!field) return undefined;
+  return dump?.mods.some((mod) => mod[field] != null)
+    ? `registry/${file}.json`
+    : undefined;
+}
+
+export function dumpModRegistryRel(
+  dump: DumpIndex | null | undefined,
+  namespace: string,
+  file: string,
+): string | undefined {
+  const field = REGISTRY_FIELD[file as keyof typeof REGISTRY_FIELD];
+  const mod = dump?.mods.find((m) => m.namespace === namespace);
+  return field && mod?.[field] ? `registry/${file}.json` : undefined;
+}
+

package/src/lib/dump/mods/load.ts

@@ -0,0 +1,60 @@
+import { asRecord, asString } from '@terrafirmagreg-team/dump/lib/dump/contract/read';
+import { dumpLocation } from '@terrafirmagreg-team/dump/lib/dump/paths';
+import { createDatasetCache, dumpReader } from '@terrafirmagreg-team/dump/lib/dump/reader';
+import { parseDumpManifest } from '@terrafirmagreg-team/dump/lib/dump/manifest/parse';
+import type { DatasetRef } from '@terrafirmagreg-team/sites/location';
+import { dumpIndexFromManifest } from '@terrafirmagreg-team/dump/lib/dump/mods/index';
+import type { DumpIndex } from '@terrafirmagreg-team/dump/lib/dump/mods/dto';
+
+const dumpCache = createDatasetCache(
+  async (dataset: DatasetRef): Promise<DumpIndex | null> => {
+    const reader = await dumpReader(dataset);
+    const manifest = parseDumpManifest(await reader.getJson('manifest.json'));
+    if (!manifest) return null;
+    return dumpIndexFromManifest(
+      manifest,
+      dumpLocation(dataset)?.root ?? dataset.id,
+    );
+  },
+);
+
+export function loadDatasetDump(dataset: DatasetRef): Promise<DumpIndex | null> {
+  return dumpCache(dataset);
+}
+
+export async function discoverDumpDatasets(
+  datasetIds: readonly string[],
+): Promise<DatasetRef[]> {
+  const refs = await Promise.all(
+    datasetIds.map(async (id) => {
+      const versions = asRecord(
+        await dumpReader({ id }).then((reader) => reader.getJson('versions.json')),
+      );
+      const latest = asString(versions?.latest);
+      if (!versions || !latest) return [];
+      const manifest = await dumpReader({ id }).then((reader) =>
+        reader.getJson('manifest.json'),
+      );
+      if (!manifest) return [];
+      const listed = Array.isArray(versions.versions)
+        ? versions.versions.filter(
+            (row): row is string => typeof row === 'string',
+          )
+        : [];
+      const published = await Promise.all(
+        listed.map(async (version): Promise<DatasetRef | null> => {
+          const ref = { id, version };
+          const ok = await dumpReader(ref).then((reader) =>
+            reader.getJson('manifest.json'),
+          );
+          return ok ? ref : null;
+        }),
+      );
+      return [{ id }, ...published.filter((row): row is DatasetRef => row !== null)];
+    }),
+  );
+  return refs.flat().sort((a, b) =>
+    a.id.localeCompare(b.id) || (a.version ?? '').localeCompare(b.version ?? ''),
+  );
+}
+