ddys-search-index 0.1.0

package/src/client.js

@@ -0,0 +1,128 @@
+import { createTimeoutSignal, joinUrl } from './utils.js';
+
+export async function fetchJson(pathname, query, config, runtime = {}) {
+  const fetchImpl = runtime.fetch || globalThis.fetch;
+  if (typeof fetchImpl !== 'function') throw new Error('A fetch implementation is required.');
+  const url = buildApiUrl(pathname, query, config);
+  const timeout = createTimeoutSignal(config.requestTimeoutMs);
+  let response;
+  try {
+    response = await fetchImpl(url, {
+      method: 'GET',
+      headers: {
+        accept: 'application/json',
+        'user-agent': config.userAgent
+      },
+      signal: timeout.signal
+    });
+  } catch (error) {
+    if (timeout.timedOut()) throw new Error(`DDYS request timed out after ${config.requestTimeoutMs}ms: ${pathname}`);
+    throw error;
+  } finally {
+    timeout.cancel();
+  }
+
+  const payload = await readJsonResponse(response);
+  if (!response.ok || payload?.success === false) {
+    throw new Error(`DDYS request failed: ${payload?.message || response.status}`);
+  }
+  return { url, payload };
+}
+
+export async function fetchSearchSource(source, config, runtime = {}) {
+  if (!source.paginated) {
+    const { url, payload } = await fetchJson(source.endpoint, { ...source.query, limit: source.limit }, config, runtime);
+    return { source, urls: [url], payloads: [payload], items: extractItems(payload) };
+  }
+
+  const payloads = [];
+  const urls = [];
+  const items = [];
+  for (let page = 1; page <= source.maxPages; page += 1) {
+    const { url, payload } = await fetchJson(source.endpoint, {
+      ...source.query,
+      page,
+      limit: source.limit
+    }, config, runtime);
+    urls.push(url);
+    payloads.push(payload);
+    const pageItems = extractItems(payload);
+    items.push(...pageItems);
+    if (!pageItems.length || pageItems.length < source.limit) break;
+    const totalPages = readTotalPages(payload);
+    if (totalPages && page >= totalPages) break;
+  }
+  return { source, urls, payloads, items };
+}
+
+export async function fetchSearchSources(sources, config, runtime = {}) {
+  const out = [];
+  for (const source of sources) out.push(await fetchSearchSource(source, config, runtime));
+  return out;
+}
+
+export function buildApiUrl(pathname, query = {}, config) {
+  const raw = String(pathname || '').trim();
+  const url = raw.startsWith('http://') || raw.startsWith('https://')
+    ? new URL(raw)
+    : new URL(joinUrl(config.apiBase, raw || '/'));
+  for (const [key, value] of Object.entries(query || {})) {
+    if (value === undefined || value === null || value === '') continue;
+    if (Array.isArray(value)) {
+      for (const item of value) url.searchParams.append(key, String(item));
+      continue;
+    }
+    url.searchParams.set(key, String(value));
+  }
+  return url.toString();
+}
+
+export function extractItems(payload) {
+  if (Array.isArray(payload)) return payload;
+  if (!payload || typeof payload !== 'object') return [];
+  for (const candidate of [
+    payload.items,
+    payload.results,
+    payload.list,
+    payload.records,
+    payload.data,
+    payload.data?.items,
+    payload.data?.results,
+    payload.data?.list,
+    payload.data?.records,
+    payload.result?.items,
+    payload.result?.results,
+    payload.result?.list,
+    payload.result?.records
+  ]) {
+    if (Array.isArray(candidate)) return candidate;
+  }
+  return [];
+}
+
+export function readTotalPages(payload) {
+  const candidates = [
+    payload?.meta?.total_pages,
+    payload?.meta?.totalPages,
+    payload?.meta?.pages,
+    payload?.pagination?.total_pages,
+    payload?.pagination?.totalPages,
+    payload?.data?.total_pages,
+    payload?.data?.totalPages
+  ];
+  for (const value of candidates) {
+    const number = Number(value);
+    if (Number.isInteger(number) && number > 0) return number;
+  }
+  return 0;
+}
+
+async function readJsonResponse(response) {
+  const text = await response.text();
+  if (!text) return {};
+  try {
+    return JSON.parse(text);
+  } catch (error) {
+    throw new Error(`Failed to parse DDYS API response as JSON: ${error.message}`);
+  }
+}

package/src/config.js

@@ -0,0 +1,212 @@
+import { hashString, normalizeBaseUrl, parseBoolean, parsePositiveInteger, sanitizeFilePart, splitList } from './utils.js';
+
+export const VERSION = '0.1.0';
+
+export const DEFAULT_CONFIG = {
+  version: VERSION,
+  apiBase: 'https://ddys.io/api/v1',
+  publicBase: 'https://ddys.io',
+  userAgent: `ddys-search-index/${VERSION}`,
+  source: 'latest',
+  sources: ['latest', 'hot'],
+  maxItems: 50,
+  maxPages: 2,
+  ttlSeconds: 600,
+  requestTimeoutMs: 10000,
+  minTokenLength: 1,
+  includeRaw: false,
+  workerCache: true,
+  fields: ['title', 'aliases', 'description', 'actors', 'directors', 'tags', 'category', 'region', 'language', 'year', 'status'],
+  facets: ['category', 'year', 'region', 'language', 'source']
+};
+
+export function loadConfigFromEnv(env = getProcessEnv()) {
+  return createConfig({}, env);
+}
+
+export function createConfig(input = {}, env = input.env || getProcessEnv()) {
+  const envConfig = readEnvConfig(env || {});
+  const merged = {
+    ...DEFAULT_CONFIG,
+    ...envConfig,
+    ...input
+  };
+
+  merged.apiBase = normalizeBaseUrl(merged.apiBase, 'apiBase');
+  merged.publicBase = normalizeBaseUrl(merged.publicBase, 'publicBase');
+  merged.maxItems = parsePositiveInteger(merged.maxItems, DEFAULT_CONFIG.maxItems, 'maxItems');
+  merged.maxPages = parsePositiveInteger(merged.maxPages, DEFAULT_CONFIG.maxPages, 'maxPages');
+  merged.ttlSeconds = parsePositiveInteger(merged.ttlSeconds, DEFAULT_CONFIG.ttlSeconds, 'ttlSeconds');
+  merged.requestTimeoutMs = parsePositiveInteger(merged.requestTimeoutMs, DEFAULT_CONFIG.requestTimeoutMs, 'requestTimeoutMs');
+  merged.minTokenLength = parsePositiveInteger(merged.minTokenLength, DEFAULT_CONFIG.minTokenLength, 'minTokenLength');
+  merged.includeRaw = parseBoolean(merged.includeRaw, DEFAULT_CONFIG.includeRaw);
+  merged.workerCache = parseBoolean(merged.workerCache, DEFAULT_CONFIG.workerCache);
+  merged.fields = normalizeNames(input.fields ?? envConfig.fields ?? DEFAULT_CONFIG.fields);
+  merged.facets = normalizeNames(input.facets ?? envConfig.facets ?? DEFAULT_CONFIG.facets);
+  merged.source = normalizeSource(input.source ?? envConfig.source ?? DEFAULT_CONFIG.source, merged);
+  merged.sources = normalizeSources(input.sources ?? envConfig.sources ?? [merged.source], merged);
+  return merged;
+}
+
+export function readEnvConfig(env = {}) {
+  const out = {};
+  if (env.DDYS_SEARCH_INDEX_API_BASE) out.apiBase = env.DDYS_SEARCH_INDEX_API_BASE;
+  if (env.DDYS_SEARCH_INDEX_PUBLIC_BASE) out.publicBase = env.DDYS_SEARCH_INDEX_PUBLIC_BASE;
+  if (env.DDYS_SEARCH_INDEX_USER_AGENT) out.userAgent = env.DDYS_SEARCH_INDEX_USER_AGENT;
+  if (env.DDYS_SEARCH_INDEX_SOURCE) out.source = env.DDYS_SEARCH_INDEX_SOURCE;
+  if (env.DDYS_SEARCH_INDEX_SOURCES) out.sources = env.DDYS_SEARCH_INDEX_SOURCES;
+  if (env.DDYS_SEARCH_INDEX_MAX_ITEMS) out.maxItems = env.DDYS_SEARCH_INDEX_MAX_ITEMS;
+  if (env.DDYS_SEARCH_INDEX_MAX_PAGES) out.maxPages = env.DDYS_SEARCH_INDEX_MAX_PAGES;
+  if (env.DDYS_SEARCH_INDEX_TTL_SECONDS) out.ttlSeconds = env.DDYS_SEARCH_INDEX_TTL_SECONDS;
+  if (env.DDYS_SEARCH_INDEX_TIMEOUT_MS) out.requestTimeoutMs = env.DDYS_SEARCH_INDEX_TIMEOUT_MS;
+  if (env.DDYS_SEARCH_INDEX_MIN_TOKEN_LENGTH) out.minTokenLength = env.DDYS_SEARCH_INDEX_MIN_TOKEN_LENGTH;
+  if (env.DDYS_SEARCH_INDEX_INCLUDE_RAW) out.includeRaw = env.DDYS_SEARCH_INDEX_INCLUDE_RAW;
+  if (env.DDYS_SEARCH_INDEX_WORKER_CACHE) out.workerCache = env.DDYS_SEARCH_INDEX_WORKER_CACHE;
+  if (env.DDYS_SEARCH_INDEX_FIELDS) out.fields = env.DDYS_SEARCH_INDEX_FIELDS;
+  if (env.DDYS_SEARCH_INDEX_FACETS) out.facets = env.DDYS_SEARCH_INDEX_FACETS;
+  return out;
+}
+
+export function normalizeSources(value, config = DEFAULT_CONFIG) {
+  const values = Array.isArray(value) ? value : splitList(value);
+  const sources = values.map((item) => normalizeSource(item, config)).filter(Boolean);
+  return sources.length ? sources : [normalizeSource('latest', config)];
+}
+
+export function normalizeSource(value, config = DEFAULT_CONFIG) {
+  if (!value && value !== 0) return null;
+  if (typeof value === 'object' && !Array.isArray(value)) {
+    const source = { ...value };
+    source.kind = normalizeSourceKind(source.kind || source.type || source.name || 'endpoint');
+    if (source.kind !== 'endpoint' && !source.endpoint) source.endpoint = endpointForKind(source.kind);
+    source.limit = parsePositiveInteger(source.limit ?? config.maxItems, config.maxItems, 'source.limit');
+    source.maxPages = parsePositiveInteger(source.maxPages ?? config.maxPages, config.maxPages, 'source.maxPages');
+    source.query = normalizeSourceQuery(source, config);
+    source.value = source.value || source.query?.q || '';
+    source.paginated = source.paginated ?? source.kind !== 'calendar';
+    source.label = source.label || labelForSource(source);
+    source.slug = source.slug || slugForSource(source);
+    return source;
+  }
+
+  const token = String(value || '').trim();
+  if (!token) return null;
+  const separator = token.indexOf(':');
+  const rawKind = separator >= 0 ? token.slice(0, separator) : token;
+  const rawValue = separator >= 0 ? token.slice(separator + 1) : '';
+  const kind = normalizeSourceKind(rawKind);
+  const source = {
+    kind,
+    value: rawValue,
+    endpoint: kind === 'endpoint' ? rawValue : endpointForKind(kind),
+    limit: config.maxItems,
+    maxPages: config.maxPages,
+    paginated: kind !== 'calendar'
+  };
+  source.query = normalizeSourceQuery(source, config);
+  source.label = labelForSource(source);
+  source.slug = slugForSource(source);
+  return source;
+}
+
+export function describeConfig(config) {
+  return {
+    version: config.version,
+    apiBase: config.apiBase,
+    publicBase: config.publicBase,
+    source: describeSource(config.source),
+    sources: config.sources.map((source) => describeSource(source)),
+    maxItems: config.maxItems,
+    maxPages: config.maxPages,
+    ttlSeconds: config.ttlSeconds,
+    requestTimeoutMs: config.requestTimeoutMs,
+    minTokenLength: config.minTokenLength,
+    includeRaw: config.includeRaw,
+    workerCache: config.workerCache,
+    fields: config.fields,
+    facets: config.facets
+  };
+}
+
+function normalizeNames(value) {
+  const names = (Array.isArray(value) ? value : splitList(value))
+    .map((item) => String(item || '').trim())
+    .filter(Boolean);
+  return names.length ? [...new Set(names)] : [];
+}
+
+function normalizeSourceKind(value) {
+  const kind = String(value || '').trim().toLowerCase();
+  const aliases = {
+    new: 'latest',
+    newest: 'latest',
+    popular: 'hot',
+    cal: 'calendar',
+    schedule: 'calendar',
+    find: 'search',
+    s: 'search',
+    api: 'endpoint',
+    path: 'endpoint',
+    custom: 'endpoint'
+  };
+  return aliases[kind] || kind || 'latest';
+}
+
+function endpointForKind(kind) {
+  if (kind === 'latest') return '/latest';
+  if (kind === 'hot') return '/hot';
+  if (kind === 'calendar') return '/calendar';
+  if (kind === 'search') return '/search';
+  if (kind === 'endpoint') return '';
+  throw new Error(`Unsupported source kind: ${kind}`);
+}
+
+function normalizeSourceQuery(source, config) {
+  const query = { ...(source.query || {}) };
+  if (!('limit' in query) && source.kind !== 'calendar') query.limit = source.limit || config.maxItems;
+  if (source.kind === 'search') {
+    const q = source.value || query.q || query.query || '';
+    if (!q) throw new Error('search source requires a keyword, for example search:keyword.');
+    query.q = q;
+    delete query.query;
+  }
+  if (source.kind === 'calendar') {
+    const value = String(source.value || '').trim();
+    if (/^\d{4}-\d{1,2}-\d{1,2}$/.test(value) || /^\d{4}-\d{1,2}$/.test(value)) {
+      const [year, month] = value.split('-');
+      query.year = Number(year);
+      query.month = Number(month);
+    }
+  }
+  return query;
+}
+
+function labelForSource(source) {
+  if (source.label) return source.label;
+  if (source.kind === 'search') return `search:${source.value || source.query?.q || ''}`;
+  if (source.kind === 'endpoint') return source.endpoint || 'endpoint';
+  return source.kind;
+}
+
+function slugForSource(source) {
+  if (source.kind === 'search') return `search-${hashString(source.value || source.query?.q || '')}`;
+  if (source.kind === 'endpoint') return `endpoint-${hashString(source.endpoint || source.label || '')}`;
+  return sanitizeFilePart(source.label || source.kind, source.kind || 'source');
+}
+
+function describeSource(source) {
+  return {
+    kind: source.kind,
+    label: source.label,
+    slug: source.slug,
+    endpoint: source.endpoint,
+    query: source.query,
+    limit: source.limit,
+    maxPages: source.maxPages,
+    paginated: source.paginated
+  };
+}
+
+function getProcessEnv() {
+  return globalThis.process?.env || {};
+}

package/src/index.js

@@ -0,0 +1,7 @@
+export { createConfig, describeConfig, loadConfigFromEnv } from './config.js';
+export { buildApiUrl, extractItems, fetchJson, fetchSearchSource, fetchSearchSources } from './client.js';
+export { normalizeDocument, normalizeRecords } from './normalize.js';
+export { buildSearchIndex, searchIndex, tokenize, buildFacets } from './indexer.js';
+export { buildSearchFiles, renderSearchDocs, renderSearchIndex, renderSearchMeta } from './renderers.js';
+export { buildOutputHeaders, generateSearchBundle, generateSearchIndex, readInputFile, writeSearchBundle } from './runtime.js';
+export { createWorkerHandler, handleWorkerRequest, routeForPath } from './worker.js';

package/src/indexer.js

@@ -0,0 +1,182 @@
+import { VERSION } from './config.js';
+import { normalizeRecords } from './normalize.js';
+import { hashString, normalizeWhitespace, uniqueStrings } from './utils.js';
+
+const FIELD_WEIGHTS = {
+  title: 10,
+  aliases: 8,
+  tags: 6,
+  category: 5,
+  actors: 4,
+  directors: 4,
+  region: 3,
+  language: 3,
+  year: 3,
+  status: 2,
+  description: 1,
+  text: 1
+};
+
+export function buildSearchIndex(records, config, runtime = {}) {
+  const now = new Date((runtime.now || (() => Date.now()))()).toISOString();
+  const documents = normalizeRecords(records, config);
+  const postings = {};
+  const documentTokens = {};
+
+  for (const doc of documents) {
+    const scores = buildTokenScores(doc, config);
+    documentTokens[doc.id] = Object.keys(scores);
+    for (const [token, score] of Object.entries(scores)) {
+      if (!postings[token]) postings[token] = [];
+      postings[token].push([doc.id, score]);
+    }
+  }
+
+  for (const token of Object.keys(postings)) {
+    postings[token].sort((a, b) => b[1] - a[1] || String(a[0]).localeCompare(String(b[0])));
+  }
+
+  const facets = buildFacets(documents, config.facets);
+  const suggestions = buildSuggestions(documents);
+  const index = {
+    version: VERSION,
+    generatedAt: now,
+    config: {
+      apiBase: config.apiBase,
+      publicBase: config.publicBase,
+      sources: config.sources.map((source) => ({
+        kind: source.kind,
+        label: source.label,
+        slug: source.slug,
+        endpoint: source.endpoint,
+        query: source.query
+      })),
+      fields: config.fields,
+      facets: config.facets,
+      minTokenLength: config.minTokenLength
+    },
+    stats: {
+      documents: documents.length,
+      tokens: Object.keys(postings).length,
+      suggestions: suggestions.length
+    },
+    documents,
+    index: postings,
+    documentTokens,
+    facets,
+    suggestions
+  };
+  index.checksum = hashString(JSON.stringify({ documents, postings }), 16);
+  return index;
+}
+
+export function searchIndex(searchIndexValue, query, options = {}) {
+  const tokens = tokenize(query, options);
+  const docs = searchIndexValue.documents || [];
+  const byId = new Map(docs.map((doc) => [String(doc.id), doc]));
+  const scores = new Map();
+  for (const token of tokens) {
+    const postings = searchIndexValue.index?.[token] || [];
+    for (const [id, score] of postings) {
+      const key = String(id);
+      scores.set(key, (scores.get(key) || 0) + score);
+    }
+  }
+
+  const filters = options.filters || {};
+  const items = [...scores.entries()]
+    .map(([id, score]) => ({ score, document: byId.get(id) }))
+    .filter((item) => item.document && matchesFilters(item.document, filters))
+    .sort((a, b) => b.score - a.score || itemTitle(a).localeCompare(itemTitle(b)));
+
+  const limit = Number.isInteger(options.limit) && options.limit > 0 ? options.limit : 20;
+  return {
+    query: String(query || ''),
+    tokens,
+    total: items.length,
+    items: items.slice(0, limit)
+  };
+}
+
+export function tokenize(value, options = {}) {
+  const minTokenLength = options.minTokenLength || 1;
+  const normalized = normalizeWhitespace(String(value || '').toLowerCase())
+    .normalize('NFKC')
+    .replace(/[_-]+/g, ' ');
+  const tokens = new Set();
+
+  for (const match of normalized.matchAll(/[a-z0-9]+/g)) {
+    const token = match[0];
+    if (token.length >= minTokenLength) tokens.add(token);
+  }
+
+  for (const match of normalized.matchAll(/[\u4e00-\u9fff]+/g)) {
+    const text = match[0];
+    if (text.length >= minTokenLength) tokens.add(text);
+    for (const char of text) {
+      if (char.length >= minTokenLength) tokens.add(char);
+    }
+    for (let index = 0; index < text.length - 1; index += 1) {
+      tokens.add(text.slice(index, index + 2));
+    }
+  }
+
+  return [...tokens].filter((token) => token.length >= minTokenLength);
+}
+
+export function buildFacets(documents, facetNames) {
+  const facets = {};
+  for (const name of facetNames || []) {
+    const counts = new Map();
+    for (const doc of documents) {
+      const values = Array.isArray(doc[name]) ? doc[name] : [doc[name]];
+      for (const raw of values) {
+        const value = normalizeWhitespace(raw);
+        if (!value) continue;
+        counts.set(value, (counts.get(value) || 0) + 1);
+      }
+    }
+    facets[name] = [...counts.entries()]
+      .map(([value, count]) => ({ value, count }))
+      .sort((a, b) => b.count - a.count || a.value.localeCompare(b.value));
+  }
+  return facets;
+}
+
+function buildTokenScores(doc, config) {
+  const scores = {};
+  for (const field of config.fields || []) {
+    const weight = FIELD_WEIGHTS[field] || 1;
+    const values = Array.isArray(doc[field]) ? doc[field] : [doc[field]];
+    for (const value of values) {
+      for (const token of tokenize(value, { minTokenLength: config.minTokenLength })) {
+        scores[token] = Math.min(99, (scores[token] || 0) + weight);
+      }
+    }
+  }
+  return scores;
+}
+
+function buildSuggestions(documents) {
+  return uniqueStrings(documents.flatMap((doc) => [
+    doc.title,
+    ...(doc.aliases || []),
+    ...(doc.tags || []),
+    ...(doc.actors || []),
+    ...(doc.directors || [])
+  ])).slice(0, 200);
+}
+
+function matchesFilters(doc, filters) {
+  for (const [key, expected] of Object.entries(filters || {})) {
+    if (expected === undefined || expected === null || expected === '') continue;
+    const values = Array.isArray(doc[key]) ? doc[key] : [doc[key]];
+    const wanted = Array.isArray(expected) ? expected.map(String) : [String(expected)];
+    if (!values.some((value) => wanted.includes(String(value)))) return false;
+  }
+  return true;
+}
+
+function itemTitle(item) {
+  return item.document?.title || '';
+}