@terrafirmagreg-team/dump 0.0.0-stage → 1.2.0

package/src/lib/dump/paths.ts

@@ -0,0 +1,172 @@
+import type { DatasetRef } from '@terrafirmagreg-team/sites/location';
+import { stackIdBase } from '@terrafirmagreg-team/dump/lib/dump/contract/ref';
+
+export type DumpLocation = {
+
+  root: string;
+
+  files?: Record<string, string>;
+};
+
+const dumpLocations = new Map<string, DumpLocation>();
+const dumpLocationListeners = new Set<() => void>();
+
+export function registerDumpLocation(
+  dataset: DatasetRef,
+  location: DumpLocation,
+): void {
+  dumpLocations.set(`${dataset.id}/${dataset.version ?? ''}`, location);
+  for (const listener of dumpLocationListeners) listener();
+}
+
+export function subscribeDumpLocations(listener: () => void): () => void {
+  dumpLocationListeners.add(listener);
+  return () => dumpLocationListeners.delete(listener);
+}
+
+export function dumpLocation(dataset: DatasetRef): DumpLocation | undefined {
+  return dumpLocations.get(`${dataset.id}/${dataset.version ?? ''}`);
+}
+
+export function dumpStorageUrl(storageRel: string): string {
+  const origin = (
+    process.env.NEXT_PUBLIC_DUMP_STORAGE_URL ??
+    process.env.DUMP_STORAGE_URL ??
+    ''
+  ).replace(/\/+$/, '');
+  if (!origin) {
+    throw new Error(
+      'NEXT_PUBLIC_DUMP_STORAGE_URL is not set: dump storage is required.',
+    );
+  }
+  return `${origin}/${storageRel}`;
+}
+
+export function dumpPublicUrl(
+  dataset: DatasetRef,
+  ...segments: string[]
+): string | undefined {
+  const ds = cleanDumpSegment(dataset.id);
+  if (!ds || ds.includes('/')) {
+    throw new Error(`Invalid dump dataset segment: ${dataset.id}`);
+  }
+  const rel = segments
+    .map(cleanDumpSegment)
+    .filter((s): s is string => Boolean(s))
+    .join('/');
+  const loc = dumpLocation(dataset);
+  if (!loc) return undefined;
+  const key = rel ? loc.files?.[rel] : undefined;
+  const storageRel = key ?? `${loc.root}${rel ? `/${rel}` : ''}`;
+  return dumpStorageUrl(storageRel);
+}
+
+function cleanDumpSegment(segment: string): string | null {
+  const cleaned = segment.replaceAll('\\', '/').replace(/^\/+/, '');
+  if (!cleaned || cleaned.includes('..')) return null;
+  return cleaned;
+}
+
+export function cleanDumpRel(rel: string): string | null {
+  return cleanDumpSegment(rel);
+}
+
+function dumpDirOf(rel: string): string {
+  const cleaned = cleanDumpRel(rel) ?? '';
+  const slash = cleaned.lastIndexOf('/');
+  return slash >= 0 ? cleaned.slice(0, slash) : '';
+}
+
+export function joinDumpRel(
+  baseFileRel: string,
+  childRel: string,
+): string | null {
+  const child = cleanDumpRel(childRel);
+  if (!child) return null;
+  const dir = dumpDirOf(baseFileRel);
+  return dir ? `${dir}/${child}` : child;
+}
+
+const TAG_REGISTRIES = ['items', 'blocks', 'fluids', 'entities'] as const;
+
+export function tagHashId(tagId: string): string {
+  return `#${tagId.replace(/^#+/, '')}`;
+}
+
+export function tagIdForms(tagId: string): string[] {
+  const raw = tagId.startsWith('#') ? tagId.slice(1) : tagId;
+  return [raw, `#${raw}`];
+}
+
+function tagRegistryKeys(
+  tagId: string,
+  preferredRegistry?: string,
+): string[] {
+  const raw = tagId.startsWith('#') ? tagId.slice(1) : tagId;
+  const parts = splitDumpId(raw);
+  if (!parts) return [];
+  const registries = preferredRegistry
+    ? [
+        preferredRegistry,
+        ...TAG_REGISTRIES.filter((registry) => registry !== preferredRegistry),
+      ]
+    : TAG_REGISTRIES;
+  return registries.map((registry) => `${parts.ns}:${registry}/${parts.path}`);
+}
+
+export function matchTagIndexKey(
+  entries: Record<string, unknown> | undefined,
+  tagId: string,
+  preferredRegistry?: string,
+): string | undefined {
+  if (!entries) return undefined;
+  for (const form of tagIdForms(tagId)) {
+    if (entries[form] !== undefined) return form;
+  }
+  for (const key of tagRegistryKeys(tagId, preferredRegistry)) {
+    if (entries[key] !== undefined) return key;
+  }
+  const raw = tagId.startsWith('#') ? tagId.slice(1) : tagId;
+  const parts = splitDumpId(raw);
+  if (!parts) return undefined;
+  const candidates = Object.keys(entries).filter(
+    (key) => key.startsWith(`${parts.ns}:`) && key.endsWith(`/${parts.path}`),
+  );
+  return candidates.sort()[0];
+}
+
+export function splitDumpId(
+  id: string,
+): { ns: string; path: string } | undefined {
+  const sep = id.indexOf(':');
+  if (sep < 1) return undefined;
+  const ns = id.slice(0, sep);
+  const path = id
+    .slice(sep + 1)
+    .replaceAll('\\', '/')
+    .replace(/\/{2,}/g, '/')
+    .replace(/^\/+|\/+$/g, '');
+  if (!ns || !path || ns.includes('/') || ns.includes('..') || path.includes('..')) {
+    return undefined;
+  }
+  return { ns, path };
+}
+
+export function registryId(id: string): string {
+  return stackIdBase(id);
+}
+
+export function tagPseudoId(tag: string, kind?: string): string {
+  const raw = tagIdForms(tag)[0]!;
+  const parts = splitDumpId(raw);
+  return `emi:tag/${kind ?? 'item'}/${parts ? `${parts.ns}/${parts.path}` : raw}`;
+}
+
+export function dumpRelPublicUrl(
+  dataset: DatasetRef,
+  rel: string | null | undefined,
+): string | undefined {
+  const cleaned = typeof rel === 'string' ? cleanDumpRel(rel) : null;
+  return cleaned ? dumpPublicUrl(dataset, cleaned) : undefined;
+}
+

package/src/lib/dump/reader.ts

@@ -0,0 +1,31 @@
+import { DUMP_POINTER_TTL, JsonDumpReader } from '@terrafirmagreg-team/dump/lib/dump/json-reader';
+import type { DatasetRef } from '@terrafirmagreg-team/sites/location';
+import {
+  createAsyncCache,
+  type AsyncCache,
+} from '@terrafirmagreg-team/data/lib/async-cache';
+
+export interface DumpReader {
+  getJson(path: string, priority?: 'high'): Promise<unknown | undefined>;
+  getLang(lang: string): Promise<Record<string, string>>;
+  getLangRaw(lang: string): Promise<Record<string, string>>;
+}
+
+export function createDatasetCache<A extends unknown[], V>(
+  loader: (dataset: DatasetRef, ...rest: A) => Promise<V>,
+  keyOf: (...args: [dataset: DatasetRef, ...rest: A]) => string =
+    ({ id, version }, ...rest) => JSON.stringify([id, version ?? '', ...rest]),
+): AsyncCache<[dataset: DatasetRef, ...rest: A], V> {
+  const versioned = createAsyncCache(loader, keyOf);
+  const latest = createAsyncCache(loader, keyOf, DUMP_POINTER_TTL);
+  const pick = (dataset: DatasetRef) => dataset.version ? versioned : latest;
+  const call = (dataset: DatasetRef, ...rest: A) => pick(dataset)(dataset, ...rest);
+  call.peek = (dataset: DatasetRef, ...rest: A) =>
+    pick(dataset).peek(dataset, ...rest);
+  return call;
+}
+
+export const dumpReader = createDatasetCache(
+  async (dataset: DatasetRef): Promise<DumpReader> => new JsonDumpReader(dataset),
+);
+

package/src/lib/dump-search.ts

@@ -0,0 +1,125 @@
+import { gzipSync } from 'node:zlib';
+import { AwsClient } from 'aws4fetch';
+import {
+  initAdvancedSearch,
+  type AdvancedIndex,
+} from 'fumadocs-core/search/server';
+import type { DatasetRef } from '@terrafirmagreg-team/sites/location';
+import { dumpReader } from '@terrafirmagreg-team/dump/lib/dump/reader';
+import { parseDumpManifest } from '@terrafirmagreg-team/dump/lib/dump/manifest/parse';
+import { dumpIndexFromManifest } from '@terrafirmagreg-team/dump/lib/dump/mods/index';
+import type { DumpIndex } from '@terrafirmagreg-team/dump/lib/dump/mods/dto';
+import type { DumpManifest } from '@terrafirmagreg-team/dump/lib/dump/manifest/dto';
+
+export interface DumpSearchBlob {
+  type: 'i18n';
+  data: Record<string, unknown>;
+}
+
+export type DumpSearchSource = {
+  getJson(rel: string): Promise<unknown | undefined>;
+  getLang(lang: string): Promise<Record<string, string>>;
+};
+
+export type DumpSearchIndexBuilder = (
+  dataset: DatasetRef,
+  dump: DumpIndex,
+  source: DumpSearchSource,
+) => Promise<(AdvancedIndex & { locale: string })[]>;
+
+export async function resolveDumpVersion(
+  id: string,
+  version: string | undefined,
+): Promise<string | undefined> {
+  if (version) return version;
+  const reader = await dumpReader({ id });
+  const versions = (await reader.getJson('versions.json')) as
+    | { latest?: string }
+    | undefined;
+  return versions?.latest;
+}
+
+export async function buildDumpSearchBlob(
+  id: string,
+  version: string | undefined,
+  lang: string,
+  indexBuilders: DumpSearchIndexBuilder[],
+): Promise<DumpSearchBlob> {
+  const ver = await resolveDumpVersion(id, version);
+  if (!ver) return { type: 'i18n', data: {} };
+  const reader = await dumpReader({ id, version: ver });
+  const manifest = parseDumpManifest(
+    await reader.getJson('manifest.json'),
+  ) as DumpManifest | null;
+  if (!manifest) return { type: 'i18n', data: {} };
+  const dataset = { id, version: ver };
+  const dump = dumpIndexFromManifest(manifest, `${id}/${ver}`);
+  const indexes = (
+    await Promise.all(
+      indexBuilders.map((build) => build(dataset, dump, reader)),
+    )
+  ).flat();
+  const scoped = indexes.filter((index) => index.locale === lang);
+  if (!scoped.length) return { type: 'i18n', data: {} };
+  const blob = await initAdvancedSearch({
+    indexes: scoped,
+    localeFilter: true,
+  }).export();
+  return { type: 'i18n', data: { [lang]: blob } };
+}
+
+const builds = new Map<string, Promise<DumpSearchBlob>>();
+
+export function buildDumpSearchBlobCached(
+  id: string,
+  version: string | undefined,
+  lang: string,
+  indexBuilders: DumpSearchIndexBuilder[],
+): Promise<DumpSearchBlob> {
+  const key = `${id}/${version ?? ''}/${lang}`;
+  const cached = builds.get(key);
+  if (cached) return cached;
+  const p = buildDumpSearchBlob(id, version, lang, indexBuilders);
+  p.catch(() => {
+    if (builds.get(key) === p) builds.delete(key);
+  });
+  builds.set(key, p);
+  return p;
+}
+
+function s3Client(): AwsClient | undefined {
+  const accessKeyId = process.env.DUMP_S3_ACCESS_KEY;
+  const secretAccessKey = process.env.DUMP_S3_SECRET_KEY;
+  if (!accessKeyId || !secretAccessKey) return undefined;
+  return new AwsClient({
+    accessKeyId,
+    secretAccessKey,
+    region: process.env.DUMP_S3_REGION ?? 'auto',
+    service: 's3',
+  });
+}
+
+export async function putSearchBlob(
+  id: string,
+  version: string,
+  lang: string,
+  blob: unknown,
+): Promise<boolean> {
+  const aws = s3Client();
+  const endpoint = process.env.DUMP_S3_ENDPOINT;
+  const bucket = process.env.DUMP_S3_BUCKET;
+  if (!aws || !endpoint || !bucket) return false;
+  const key = `${id}/${version}/search/${lang}.json`;
+  const url = `${endpoint.replace(/\/+$/, '')}/${bucket}/${key}`;
+  const res = await aws.fetch(url, {
+    method: 'PUT',
+    body: gzipSync(JSON.stringify(blob)),
+    headers: {
+      'Content-Type': 'application/json',
+      'Content-Encoding': 'gzip',
+      'Cache-Control': 'public, max-age=3600',
+    },
+  });
+  return res.ok;
+}
+

package/src/lib/snbt.ts

@@ -0,0 +1,247 @@
+const WS = /[ \t\n\r]/;
+
+const SAFE_KEY = /^[A-Za-z0-9_.\-+]+$/;
+const SAFE_VALUE = /^[A-Za-z_][A-Za-z0-9_.\-+]*$/;
+const LITERAL_WORD = /^(true|false|null|nan|infinity)$/i;
+const MAX_SAFE = 2 ** 53;
+
+export function formatNbt(value: unknown): string | undefined {
+  if (value === undefined) return undefined;
+  return writeNbtValue(value);
+}
+
+function writeNbtValue(value: unknown): string {
+  if (value === null || value === undefined) return 'null';
+  if (typeof value === 'boolean') return value ? 'true' : 'false';
+  if (typeof value === 'number') return writeNbtNumber(value);
+  if (typeof value === 'string') return writeNbtString(value);
+  if (Array.isArray(value)) return `[${value.map(writeNbtValue).join(',')}]`;
+  if (typeof value === 'object') {
+    const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) =>
+      a < b ? -1 : a > b ? 1 : 0,
+    );
+    return `{${entries.map(([k, v]) => `${writeNbtKey(k)}:${writeNbtValue(v)}`).join(',')}}`;
+  }
+  return 'null';
+}
+
+function writeNbtKey(key: string): string {
+  return SAFE_KEY.test(key) ? key : JSON.stringify(key);
+}
+
+function writeNbtString(value: string): string {
+  return SAFE_VALUE.test(value) && !LITERAL_WORD.test(value)
+    ? value
+    : JSON.stringify(value);
+}
+
+function writeNbtNumber(value: number): string {
+  if (!Number.isFinite(value)) return '0';
+  if (Number.isInteger(value) && Math.abs(value) <= MAX_SAFE) return value.toFixed(0);
+  return String(value);
+}
+
+export function formatNbtPretty(value: unknown): string | undefined {
+  if (value === undefined) return undefined;
+  return prettyNbtValue(value, 0);
+}
+
+function prettyNbtValue(value: unknown, depth: number): string {
+  const pad = '  '.repeat(depth);
+  const padIn = `${pad}  `;
+  if (value !== null && typeof value === 'object') {
+    if (Array.isArray(value)) {
+      if (!value.length) return '[]';
+      if (value.every((v) => v === null || typeof v !== 'object')) {
+        return `[${value.map(writeNbtValue).join(', ')}]`;
+      }
+      return `[\n${value
+        .map((v) => `${padIn}${prettyNbtValue(v, depth + 1)}`)
+        .join(',\n')}\n${pad}]`;
+    }
+    const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) =>
+      a < b ? -1 : a > b ? 1 : 0,
+    );
+    if (!entries.length) return '{}';
+    return `{\n${entries
+      .map(([k, v]) => `${padIn}${writeNbtKey(k)}: ${prettyNbtValue(v, depth + 1)}`)
+      .join(',\n')}\n${pad}}`;
+  }
+  return writeNbtValue(value);
+}
+
+export function parseSnbt(text: string): unknown {
+  const src = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+  const parser = { src, i: 0 };
+  const value = readValue(parser);
+  skip(parser);
+  if (parser.i < parser.src.length) {
+    throw new Error(`SNBT trailing data at ${parser.i}`);
+  }
+  return value;
+}
+
+type Parser = { src: string; i: number };
+
+function peek(p: Parser): string {
+  return p.src[p.i] ?? '';
+}
+
+function eof(p: Parser): boolean {
+  return p.i >= p.src.length;
+}
+
+function skip(p: Parser): void {
+  while (!eof(p)) {
+    const c = peek(p);
+    if (WS.test(c) || c === ',') {
+      p.i++;
+      continue;
+    }
+    if (c === '/' && p.src[p.i + 1] === '/') {
+      p.i += 2;
+      while (!eof(p) && peek(p) !== '\n') p.i++;
+      continue;
+    }
+    break;
+  }
+}
+
+function readValue(p: Parser): unknown {
+  skip(p);
+  const c = peek(p);
+  if (!c) throw new Error('SNBT unexpected end');
+  if (c === '{') return readCompound(p);
+  if (c === '[') return readList(p);
+  if (c === '"') return readString(p);
+  return readBare(p);
+}
+
+function readCompound(p: Parser): Record<string, unknown> {
+  p.i++;
+  const out: Record<string, unknown> = {};
+  while (true) {
+    skip(p);
+    if (peek(p) === '}') {
+      p.i++;
+      return out;
+    }
+    if (eof(p)) throw new Error('SNBT unclosed compound');
+    const key = peek(p) === '"' ? readString(p) : readUnquoted(p);
+    skip(p);
+    if (peek(p) !== ':') throw new Error(`SNBT expected ':' after key ${key}`);
+    p.i++;
+    out[key] = readValue(p);
+  }
+}
+
+function readList(p: Parser): unknown[] {
+  p.i++;
+  skip(p);
+  if (/^[BILbils][ \t]*;/.test(p.src.slice(p.i, p.i + 8))) {
+    p.i++;
+    skip(p);
+    if (peek(p) === ';') p.i++;
+  }
+  const out: unknown[] = [];
+  while (true) {
+    skip(p);
+    if (peek(p) === ']') {
+      p.i++;
+      return out;
+    }
+    if (eof(p)) throw new Error('SNBT unclosed list');
+    out.push(readValue(p));
+  }
+}
+
+function readString(p: Parser): string {
+  p.i++;
+  let out = '';
+  while (!eof(p)) {
+    const c = peek(p);
+    p.i++;
+    if (c === '"') return out;
+    if (c !== '\\') {
+      out += c;
+      continue;
+    }
+    const next = peek(p);
+    p.i++;
+    switch (next) {
+      case 'n':
+        out += '\n';
+        break;
+      case 't':
+        out += '\t';
+        break;
+      case 'r':
+        out += '\r';
+        break;
+      case 'b':
+        out += '\b';
+        break;
+      case 'f':
+        out += '\f';
+        break;
+      case 'u': {
+        const hex = p.src.slice(p.i, p.i + 4);
+        if (/^[0-9a-fA-F]{4}$/.test(hex)) {
+          out += String.fromCharCode(Number.parseInt(hex, 16));
+          p.i += 4;
+        } else {
+          out += 'u';
+        }
+        break;
+      }
+      case '"':
+      case '\\':
+        out += next;
+        break;
+      default:
+        out += next;
+    }
+  }
+  throw new Error('SNBT unclosed string');
+}
+
+function readUnquoted(p: Parser): string {
+  const start = p.i;
+  while (!eof(p)) {
+    const c = peek(p);
+    if (
+      c === ':' ||
+      c === '{' ||
+      c === '}' ||
+      c === '[' ||
+      c === ']' ||
+      c === ',' ||
+      WS.test(c)
+    ) {
+      break;
+    }
+    p.i++;
+  }
+  if (p.i === start) throw new Error(`SNBT expected key at ${p.i}`);
+  return p.src.slice(start, p.i);
+}
+
+function readBare(p: Parser): string | number | boolean {
+  const raw = readUnquoted(p);
+  if (raw === 'true') return true;
+  if (raw === 'false') return false;
+  const num = parseSnbtNumber(raw);
+  return num === undefined ? raw : num;
+}
+
+const NUM = /^-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?[bBsSlLfFdD]?$/;
+
+function parseSnbtNumber(raw: string): number | undefined {
+  if (!NUM.test(raw)) return undefined;
+  const last = raw[raw.length - 1];
+  const core =
+    last && /[bBsSlLfFdD]/.test(last) && !/\d/.test(last) ? raw.slice(0, -1) : raw;
+  const value = Number(core);
+  return Number.isFinite(value) ? value : undefined;
+}
+

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.