@terrafirmagreg-team/kit 0.1.0 → 1.2.0

package/src/lib/datasets.ts

@@ -3,8 +3,10 @@ import { dynamicIconImports } from 'lucide-react/dynamic';
 import { sectionConfig } from './config';
 import { sections } from './sources';
 import { datasetId } from './tree';
+import { createAsyncCache } from '@terrafirmagreg-team/data/lib/async-cache';
 import { loadLocaleRegistry, sortLangCodes } from '@terrafirmagreg-team/translate/server';
-import { parseDatasetRef } from './location';
+import type { DatasetRef } from '@terrafirmagreg-team/sites/location';
+import { sectionTabs } from '@terrafirmagreg-team/sites/sections';
 import type {
   DatasetCatalog,
   SectionDataset,
@@ -14,42 +16,53 @@ import type { SectionDump } from './config';
 import type { IconName } from 'lucide-react/dynamic';
 import type { Lang } from '@terrafirmagreg-team/translate/types/locale';
 
-const sectionMeta = new Map<SectionKey, Map<string, Record<string, unknown>>>();
+const sectionMeta = new Map<
+  SectionKey,
+  Promise<Map<string, Record<string, unknown>>>
+>();
 
 function staticSectionMeta(
   section: SectionKey,
-): Map<string, Record<string, unknown>> {
+): Promise<Map<string, Record<string, unknown>>> {
   let meta = sectionMeta.get(section);
   if (meta) return meta;
-  meta = new Map();
-  const source = sections[section]?.source;
-  if (source) {
-    for (const node of source.getPageTree().children) {
-      if (node.type !== 'folder') continue;
-      const id = datasetId(node);
-      const data = id
-        ? (source.getNodeMeta?.(node) as { data?: Record<string, unknown> })
-            ?.data
-        : undefined;
-      if (id && data) meta.set(id, data);
+  meta = (async () => {
+    const map = new Map<string, Record<string, unknown>>();
+    const source = await sections[section]?.getSource();
+    if (source) {
+      for (const node of source.getPageTree().children) {
+        if (node.type !== 'folder') continue;
+        const id = datasetId(node);
+        const data = id
+          ? (source.getNodeMeta?.(node) as { data?: Record<string, unknown> })
+              ?.data
+          : undefined;
+        if (id && data) map.set(id, data);
+      }
     }
-  }
+    return map;
+  })();
   sectionMeta.set(section, meta);
+  meta.catch(() => {
+    if (sectionMeta.get(section) === meta) sectionMeta.delete(section);
+  });
   return meta;
 }
 
-function contentSectionDatasets(section: SectionKey): SectionDataset[] {
-  const source = sections[section]?.source;
+async function contentSectionDatasets(
+  section: SectionKey,
+): Promise<SectionDataset[]> {
+  const source = await sections[section]?.getSource();
   if (!source) return [];
   const languages = new Map<string, Set<string>>();
   for (const page of source.getPages()) {
-    const [dataset, lang] = page.slugs;
+    const [dataset, lang] = page.path.split('/');
     if (!dataset || !lang) continue;
     let set = languages.get(dataset);
     if (!set) languages.set(dataset, (set = new Set()));
     set.add(lang);
   }
-  const meta = staticSectionMeta(section);
+  const meta = await staticSectionMeta(section);
   return [...languages.keys()].map((id) => ({
     id,
     languages: datasetLanguages([...(languages.get(id) ?? [])]),
@@ -58,11 +71,11 @@ function contentSectionDatasets(section: SectionKey): SectionDataset[] {
   }));
 }
 
-function datasetMeta(
+async function datasetMeta(
   section: SectionKey,
   id: string,
-): Omit<SectionDataset, 'id' | 'languages'> {
-  const meta = staticSectionMeta(section).get(id);
+): Promise<Omit<SectionDataset, 'id' | 'languages'>> {
+  const meta = (await staticSectionMeta(section)).get(id);
   return meta ? metaFields(meta) : {};
 }
 
@@ -89,7 +102,23 @@ function datasetLanguages(languages: readonly string[]): Lang[] {
   return [loadLocaleRegistry().defaultLanguage as Lang];
 }
 
-const bySection = new Map<SectionKey, Promise<SectionDataset[]>>();
+const DUMP_DATASETS_TTL = 60_000;
+
+const dumpSectionDatasets = createAsyncCache(
+  async (section: SectionKey, hasContent: (dump: SectionDump) => boolean) =>
+    dumpDatasets(section, hasContent),
+  (section) => section,
+  DUMP_DATASETS_TTL,
+);
+
+const contentSectionDatasetsCached = createAsyncCache(
+  async (section: SectionKey): Promise<SectionDataset[]> => {
+    const rows = await contentSectionDatasets(section);
+    rows.sort(datasetOrder);
+    return rows;
+  },
+  (section) => section,
+);
 
 function datasetOrder(a: SectionDataset, b: SectionDataset): number {
   const cfg = sectionConfig().datasetConfig;
@@ -110,6 +139,7 @@ function compareVersions(a: string, b: string): number {
 }
 
 async function dumpDatasets(
+  section: SectionKey,
   hasContent: (dump: SectionDump) => boolean,
 ): Promise<SectionDataset[]> {
   const dump = sectionConfig().dump;
@@ -117,17 +147,18 @@ async function dumpDatasets(
   const refs = await dump.discoverDatasets();
   const versions = new Map<string, string[]>();
   for (const ref of refs) {
-    const { id, version } = parseDatasetRef(ref);
+    const { id, version } = ref;
     if (version) {
       const list = versions.get(id) ?? [];
       list.push(version);
       versions.set(id, list);
     }
   }
-  const ids = refs.filter((ref) => !parseDatasetRef(ref).version);
+  const ids = refs.filter((ref) => !ref.version);
   const usable = await Promise.all(
-    ids.map(async (id) => {
-      const index = await dump.loadDataset(id);
+    ids.map(async (ref) => {
+      const { id } = ref;
+      const index = await dump.loadDataset(ref);
       if (!index || !hasContent(index)) return null;
       return {
         id,
@@ -135,37 +166,34 @@ async function dumpDatasets(
       };
     }),
   );
-  return usable
-    .filter((row): row is { id: string; languages: Lang[] } => row !== null)
-    .map(({ id, languages }) => ({
-      id,
-      languages,
-      ...(versions.get(id)?.length
-        ? { versions: versions.get(id)!.sort(compareVersions).reverse() }
-        : {}),
-      ...datasetMeta('docs', id),
-      ...datasetAppearance(id),
-    }))
-    .sort(datasetOrder);
+  return (
+    await Promise.all(
+      usable
+        .filter((row): row is { id: string; languages: Lang[] } => row !== null)
+        .map(async ({ id, languages }) => ({
+          id,
+          languages,
+          ...(versions.get(id)?.length
+            ? { versions: versions.get(id)!.sort(compareVersions).reverse() }
+            : {}),
+          ...(await datasetMeta(section, id).then((meta) =>
+            Object.values(meta).some((value) => value !== undefined)
+              ? meta
+              : datasetMeta('docs', id),
+          )),
+          ...datasetAppearance(id),
+        })),
+      )
+  ).sort(datasetOrder);
 }
 
 export function sectionDatasets(
   section: SectionKey,
 ): Promise<SectionDataset[]> {
-  let cached = bySection.get(section);
-  if (!cached) {
-    const hasContent = sectionConfig().dump?.hasContent[section];
-    if (hasContent) {
-      cached = dumpDatasets(hasContent);
-      bySection.set(section, cached);
-      return cached;
-    }
-    const rows = contentSectionDatasets(section);
-    rows.sort(datasetOrder);
-    cached = Promise.resolve(rows);
-    bySection.set(section, cached);
-  }
-  return cached;
+  const hasContent = sectionConfig().dump?.hasContent[section];
+  return hasContent
+    ? dumpSectionDatasets(section, hasContent)
+    : contentSectionDatasetsCached(section);
 }
 
 export async function defaultDataset(
@@ -176,36 +204,38 @@ export async function defaultDataset(
 
 export function findDataset(
   datasets: readonly SectionDataset[],
-  ref: string,
+  ref: DatasetRef,
 ): SectionDataset | undefined {
-  const { id, version } = parseDatasetRef(ref);
+  const { id, version } = ref;
   const meta = datasets.find((row) => row.id === id);
   if (!meta) return undefined;
   if (version && !meta.versions?.includes(version)) return undefined;
   return meta;
 }
 
-export async function resolveDatasetRef(
+export async function resolveDatasetMeta(
   section: SectionKey,
-  ref: string,
+  ref: DatasetRef,
 ): Promise<SectionDataset | undefined> {
   return findDataset(await sectionDatasets(section), ref);
 }
 
 export async function datasetSlugs(section: SectionKey): Promise<string[]> {
-  return (await sectionDatasets(section)).flatMap(({ id, versions }) => [
-    id,
-    ...(versions ?? []).map((version) => `${id}@${version}`),
-  ]);
+  return (await sectionDatasets(section)).map(({ id }) => id);
 }
 
-let catalogCached: Promise<DatasetCatalog> | undefined;
+const datasetCatalogCached = createAsyncCache(
+  async (): Promise<DatasetCatalog> =>
+    Promise.all(
+      sectionTabs.map(
+        async ({ key }) => [key, await sectionDatasets(key)] as const,
+      ),
+    ).then((rows) => Object.fromEntries(rows) as DatasetCatalog),
+  () => 'catalog',
+  DUMP_DATASETS_TTL,
+);
 
 export function datasetCatalog(): Promise<DatasetCatalog> {
-  catalogCached ??= Promise.all(
-    sectionConfig().sectionTabs.map(
-      async ({ key }) => [key, await sectionDatasets(key)] as const,
-    ),
-  ).then((rows) => Object.fromEntries(rows) as DatasetCatalog);
-  return catalogCached;
+  return datasetCatalogCached();
 }
+

package/src/lib/landing.ts

@@ -16,16 +16,17 @@ export function getDatasetLanding(
 ): DatasetLanding | undefined {
   if (!slug || slug.length !== 1) return undefined;
   const [ref] = slug;
-  const meta = findDataset(datasets, ref);
+  const meta = findDataset(datasets, { id: ref });
   if (!meta) return undefined;
   const { messages, defaultLanguage } = loadLocaleRegistry();
   const table = messages[lang] ?? messages[defaultLanguage] ?? {};
   return {
     title: table[`sections.${section}`] ?? section,
     description: table[`sections.desc.${section}`],
-    dataset: ref,
+    dataset: { id: ref },
     datasetTitle: meta.title ?? meta.id,
     lang,
+    ...(meta.versions ? { versions: meta.versions } : {}),
   };
 }
 
@@ -44,3 +45,4 @@ export async function generateDatasetLandingParams(
 ): Promise<{ slug: string[] }[]> {
   return (await datasetSlugs(section)).map((ref) => ({ slug: [ref] }));
 }
+

package/src/lib/layout.ts

@@ -13,3 +13,4 @@ export function baseOptions(): BaseLayoutProps {
     slots: { navTitle: NavTitle },
   };
 }
+

package/src/lib/locale/renderer.ts

@@ -0,0 +1,49 @@
+import type { ReactNode } from 'react';
+import type { Lang } from '@terrafirmagreg-team/translate/types/locale';
+
+export type LocaleScope = { section: string; slug?: string[] };
+
+export type LocaleResolve = (
+  slug: string[] | undefined,
+  lang: Lang,
+) => Promise<{ page: unknown; isFallback: boolean } | undefined>;
+
+export type LocaleRender = (
+  page: unknown,
+  slug: string[] | undefined,
+  lang: Lang,
+) => ReactNode | Promise<ReactNode>;
+
+export type LocaleLoadResult = { node: ReactNode; missing: boolean } | null;
+
+export type LocaleLoader = (
+  scope: LocaleScope,
+  lang: Lang,
+) => Promise<LocaleLoadResult>;
+
+const renderers = new Map<
+  string,
+  { resolve: LocaleResolve; render: LocaleRender }
+>();
+
+export function registerLocaleRenderer(
+  section: string,
+  entry: { resolve: LocaleResolve; render: LocaleRender },
+): void {
+  renderers.set(section, entry);
+}
+
+export function localeRenderer(section: string) {
+  return renderers.get(section);
+}
+
+let loader: LocaleLoader | undefined;
+
+export function registerLocaleLoader(next: LocaleLoader): void {
+  loader = next;
+}
+
+export function localeLoader(): LocaleLoader | undefined {
+  return loader;
+}
+

package/src/lib/order.ts

@@ -86,3 +86,4 @@ export function orderPlugin(): LoaderPlugin {
     },
   };
 }
+

package/src/lib/page.ts

@@ -52,7 +52,7 @@ export async function generateDatasetParams(
   section: SectionKey,
   source: SectionSource,
 ): Promise<{ slug: string[] }[]> {
-  const params = generateParams(source);
+  const params = generatePageParams(source);
   const seen = new Set(params.map((row) => row.slug.join('/')));
   for (const { id } of await sectionDatasets(section)) {
     if (!seen.has(id)) {
@@ -63,7 +63,7 @@ export async function generateDatasetParams(
   return params.length ? params : [{ slug: ['_'] }];
 }
 
-function generateParams(source: SectionSource): { slug: string[] }[] {
+export function generatePageParams(source: SectionSource): { slug: string[] }[] {
   const languages = [...sectionConfig().knownLanguages];
   const seen = new Set<string>();
   const params: { slug: string[] }[] = [];
@@ -106,3 +106,4 @@ export async function sectionMetadata(
     },
   };
 }
+

package/src/lib/pin-tree.ts

@@ -1,11 +1,14 @@
 import type * as PageTree from 'fumadocs-core/page-tree';
 
-function pinUrl(url: string, from: string, to: string): string {
+function pinUrl(url: string, from: string, version: string): string {
   const next = url[from.length];
-  return url.startsWith(from) &&
-    (next === undefined || next === '/' || next === '?' || next === '#')
-    ? `${to}${url.slice(from.length)}`
-    : url;
+  if (!url.startsWith(from) ||
+      (next !== undefined && next !== '/' && next !== '?' && next !== '#')) return url;
+  const [pathQuery, hash] = url.split('#');
+  const [path, query] = pathQuery.split('?');
+  const params = new URLSearchParams(query);
+  params.set('version', version);
+  return `${path}?${params}${hash === undefined ? '' : `#${hash}`}`;
 }
 
 export function pinTreeDataset(
@@ -15,16 +18,15 @@ export function pinTreeDataset(
   version: string,
 ): PageTree.Root {
   const from = `/${section}/${datasetId}`;
-  const to = `${from}@${version}`;
   const pinNode = (node: PageTree.Node): PageTree.Node => {
     if (node.type === 'page') {
-      return { ...node, url: pinUrl(node.url, from, to) };
+      return { ...node, url: pinUrl(node.url, from, version) };
     }
     if (node.type === 'folder') {
       return {
         ...node,
         index: node.index
-          ? { ...node.index, url: pinUrl(node.index.url, from, to) }
+          ? { ...node.index, url: pinUrl(node.index.url, from, version) }
           : node.index,
         children: node.children.map(pinNode),
       };
@@ -33,3 +35,4 @@ export function pinTreeDataset(
   };
   return { ...tree, children: tree.children.map(pinNode) };
 }
+

package/src/lib/preview/lookup.ts

@@ -1,10 +1,10 @@
 import 'server-only';
 import { resolveLang } from '@terrafirmagreg-team/translate/lib/code';
 import { loadLocaleRegistry } from '@terrafirmagreg-team/translate/server';
-import { sectionConfig } from '../config';
-import { isSectionKey } from '../location';
-import { resolvePage } from '../page';
-import { sections } from '../sources';
+import { sectionConfig } from '@terrafirmagreg-team/kit/lib/config';
+import { isSectionKey } from '@terrafirmagreg-team/sites/location';
+import { resolvePage } from '@terrafirmagreg-team/kit/lib/page';
+import { sections } from '@terrafirmagreg-team/kit/lib/sources';
 import type {
   DatasetCatalog,
   SectionKey,
@@ -37,7 +37,9 @@ export function lookupPagePreview(
     dataset?.languages ?? [...sectionConfig().knownLanguages],
     defaultLanguage,
   );
-  const resolved = resolvePage(section.source, parts.slice(1), pageLang);
+  const source = section.getSourceSync?.();
+  if (!source) return undefined;
+  const resolved = resolvePage(source, parts.slice(1), pageLang);
   const page = resolved?.page as SectionPage | undefined;
   if (!page) return undefined;
   return {
@@ -46,3 +48,4 @@ export function lookupPagePreview(
     icon: dataset?.icon ? { type: 'icon', name: dataset.icon } : undefined,
   };
 }
+

package/src/lib/section/schema.ts

@@ -1,11 +1,7 @@
 import { metaSchema, pageSchema } from 'fumadocs-core/source/schema';
 import { z, type ZodRawShape } from 'zod';
 
-export function sectionDocsConfig(
-  dir: string,
-  schemaFields: ZodRawShape = {},
-): {
-  dir: string;
+export function sectionDocsConfig(schemaFields: ZodRawShape = {}): {
   docs: {
     schema: ReturnType<typeof pageSchema.extend>;
     postprocess: { includeProcessedMarkdown: true };
@@ -13,7 +9,6 @@ export function sectionDocsConfig(
   meta: { schema: typeof metaSchema };
 } {
   return {
-    dir,
     docs: {
       schema: pageSchema.extend({
         order: z.number().optional(),
@@ -34,3 +29,4 @@ export function sectionDocsConfig(
     },
   };
 }
+

package/src/lib/section/source.ts

@@ -1,40 +1,77 @@
-import { llms, loader, type StaticSource } from 'fumadocs-core/source';
+import {
+  dynamicLoader,
+  llms,
+  type DynamicSource,
+  type LoaderConfig,
+  type LoaderOutput,
+} from 'fumadocs-core/source';
 import { sectionDatasets } from '@terrafirmagreg-team/kit/lib/datasets';
+import { sectionConfig } from '@terrafirmagreg-team/kit/lib/config';
 import { orderPlugin } from '@terrafirmagreg-team/kit/lib/order';
-import { renderSectionMarkdown } from '@terrafirmagreg-team/kit/lib/markdown';
+import { renderSectionMarkdown } from '@terrafirmagreg-team/mdx/lib/markdown';
 import { localizedDatasetTrees } from '@terrafirmagreg-team/kit/lib/tree';
 import type { SectionTrees } from '@terrafirmagreg-team/kit/types';
+import type { SectionKey } from '@terrafirmagreg-team/sites/location';
 
-type SectionSource = StaticSource;
+const LOCALE_SEGMENT = /^[a-z]{2,3}_[a-z]{2,4}$/;
 
-export function createSectionSource<S extends SectionSource>(options: {
+export function createSectionSource(options: {
   route: string;
 
-  section: string;
-  source: S;
+  section: SectionKey;
+  source: DynamicSource;
 
   postTrees?: (trees: SectionTrees) => SectionTrees;
 }) {
-  const { route, section, source: fumadocsSource, postTrees } = options;
-  const source = loader({
+  const { route, section, source: dynamicSource, postTrees } = options;
+  const source: DynamicSource = {
+    ...dynamicSource,
+    files: async () => {
+      const known = new Set(sectionConfig().knownLanguages);
+      return (await dynamicSource.files()).filter(
+        (file) =>
+          !file.path
+            .split('/')
+            .some((seg) => LOCALE_SEGMENT.test(seg) && !known.has(seg)),
+      );
+    },
+  };
+  const loader = dynamicLoader(source, {
     baseUrl: route,
-    source: fumadocsSource,
     plugins: [orderPlugin()],
   });
-  const markdown = llms(source as never, {
-    renderPage: renderSectionMarkdown as (
-      page: unknown,
-    ) => Promise<string>,
-  }) as { page(page: unknown): Promise<string> };
+  let resolved: LoaderOutput<LoaderConfig> | undefined;
+  const getSource = async () => {
+    resolved = (await loader.get()) as unknown as LoaderOutput<LoaderConfig>;
+    return resolved;
+  };
+  const getSourceSync = () => resolved;
+  const markdown = {
+    page: async (page: unknown) =>
+      llms((await getSource()) as never, {
+        renderPage: renderSectionMarkdown as (
+          page: unknown,
+        ) => Promise<string>,
+      }).page(page as never),
+  };
 
-  let cached: Promise<SectionTrees> | undefined;
-  const trees = (): Promise<SectionTrees> => {
-    cached ??= sectionDatasets(section).then((datasets) => {
-      const built = localizedDatasetTrees(source, datasets);
-      return postTrees ? postTrees(built) : built;
+  let cached:
+    | { source: LoaderOutput<LoaderConfig>; trees: Promise<SectionTrees> }
+    | undefined;
+  const trees = (): Promise<SectionTrees> =>
+    getSource().then((source) => {
+      if (cached?.source === source) return cached.trees;
+      const built = (async () => {
+        const result = localizedDatasetTrees(
+          source,
+          await sectionDatasets(section),
+        );
+        return postTrees ? postTrees(result) : result;
+      })();
+      cached = { source, trees: built };
+      return built;
     });
-    return cached;
-  };
 
-  return { route, source, markdown, trees };
+  return { route, getSource, getSourceSync, markdown, trees };
 }
+

package/src/lib/sources.ts

@@ -19,3 +19,4 @@ export const sections: Partial<Record<SectionKey, SectionSourceEntry>> =
           : undefined,
     },
   );
+

package/src/lib/tree.ts

@@ -58,3 +58,4 @@ export function localizedDatasetTrees(
   }
   return trees;
 }
+

package/src/lib/urls.ts

@@ -14,3 +14,4 @@ export function getDatasetImageUrl(section: string, dataset: string) {
     url: createGetUrl(sectionConfig().previewRoute)(segments),
   };
 }
+

package/src/locales/en_us.json

@@ -1,10 +1,7 @@
-{
-  "ui": {
-    "select_section": "Select section",
-    "select_dataset": "Select dataset",
-    "view_markdown": "View as Markdown",
-    "copy_error": "Failed to copy",
-    "search_everywhere": "Everywhere",
-    "version_latest": "latest"
-  }
-}
+{
+  "Failed to copy(ui.copy_error)": "Failed to copy",
+  "Select dataset(ui.select_dataset)": "Select dataset",
+  "Select section(ui.select_section)": "Select section",
+  "View as Markdown(ui.view_markdown)": "View as Markdown",
+  "latest(ui.version_latest)": "latest"
+}

package/src/locales/ru_ru.json

@@ -1,10 +1,7 @@
-{
-  "ui": {
-    "select_section": "Выберите раздел",
-    "select_dataset": "Выберите набор данных",
-    "view_markdown": "Просмотр Markdown",
-    "copy_error": "Не удалось скопировать",
-    "search_everywhere": "Везде",
-    "version_latest": "последняя"
-  }
-}
+{
+  "Failed to copy(ui.copy_error)": "Не удалось скопировать",
+  "Select dataset(ui.select_dataset)": "Выберите набор данных",
+  "Select section(ui.select_section)": "Выберите раздел",
+  "View as Markdown(ui.view_markdown)": "Просмотр Markdown",
+  "latest(ui.version_latest)": "последняя"
+}

package/src/locales.ts

@@ -3,3 +3,4 @@ import { createLocaleCatalog } from '@terrafirmagreg-team/translate/lib/locales'
 export const locales = createLocaleCatalog(
   import.meta.glob('./locales/*.json', { eager: true, import: 'default' }),
 );
+