ddys-search-index 0.1.0

package/src/normalize.js

@@ -0,0 +1,203 @@
+import {
+  compactObject,
+  firstValue,
+  hashString,
+  joinUrl,
+  normalizeWhitespace,
+  stripHtml,
+  toArray,
+  uniqueStrings
+} from './utils.js';
+
+export function normalizeRecords(records, config) {
+  const out = [];
+  const seen = new Set();
+  for (const record of records || []) {
+    const item = record?.item ?? record;
+    const source = record?.source || record?.sourceConfig || null;
+    const doc = normalizeDocument(item, source, config);
+    if (!doc?.id || !doc.title) continue;
+    const key = doc.id || doc.url || doc.title.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(doc);
+  }
+  return out;
+}
+
+export function normalizeDocument(item, source, config) {
+  if (!item || typeof item !== 'object') return null;
+
+  const title = firstValue(
+    item.title,
+    item.name,
+    item.vod_name,
+    item.post_title,
+    item.original_title,
+    item.originalName,
+    item.cn_name
+  );
+  if (!title) return null;
+
+  const aliases = uniqueStrings([
+    ...toArray(item.aliases),
+    ...toArray(item.alias),
+    ...toArray(item.sub_title),
+    ...toArray(item.vod_sub),
+    ...toArray(item.original_name),
+    ...toArray(item.en_title),
+    ...toArray(item.names)
+  ]).filter((value) => value.toLowerCase() !== title.toLowerCase());
+
+  const description = normalizeWhitespace(stripHtml(firstValue(
+    item.description,
+    item.desc,
+    item.summary,
+    item.content,
+    item.vod_content,
+    item.excerpt,
+    item.intro
+  )));
+
+  const actors = uniqueStrings([
+    ...toArray(item.actors),
+    ...toArray(item.actor),
+    ...toArray(item.cast),
+    ...toArray(item.vod_actor),
+    ...toArray(item.stars)
+  ]);
+
+  const directors = uniqueStrings([
+    ...toArray(item.directors),
+    ...toArray(item.director),
+    ...toArray(item.vod_director)
+  ]);
+
+  const category = uniqueStrings([
+    ...toArray(item.category),
+    ...toArray(item.categories),
+    ...toArray(item.type),
+    ...toArray(item.type_name),
+    ...toArray(item.vod_class),
+    ...toArray(item.class)
+  ]);
+
+  const tags = uniqueStrings([
+    ...toArray(item.tags),
+    ...toArray(item.tag),
+    ...toArray(item.genres),
+    ...toArray(item.genre),
+    ...category
+  ]);
+
+  const region = uniqueStrings([
+    ...toArray(item.region),
+    ...toArray(item.area),
+    ...toArray(item.country),
+    ...toArray(item.vod_area)
+  ]);
+
+  const language = uniqueStrings([
+    ...toArray(item.language),
+    ...toArray(item.lang),
+    ...toArray(item.vod_lang)
+  ]);
+
+  const id = normalizeId(item, title, source);
+  const url = buildDocumentUrl(item, config);
+  const poster = buildAssetUrl(firstValue(
+    item.poster,
+    item.cover,
+    item.image,
+    item.pic,
+    item.vod_pic,
+    item.thumbnail
+  ), config);
+
+  const year = normalizeYear(firstValue(item.year, item.vod_year, item.release_year, item.date, item.release_date));
+  const date = normalizeDate(firstValue(item.date, item.pubdate, item.updated_at, item.update_time, item.vod_time, item.created_at));
+  const score = normalizeNumber(firstValue(item.score, item.rating, item.rate, item.vod_score));
+  const status = firstValue(item.status, item.vod_remarks, item.remarks, item.update_status);
+  const sourceName = source?.label || source?.kind || firstValue(item.source, item.source_name) || 'local';
+  const text = uniqueStrings([
+    title,
+    ...aliases,
+    description,
+    ...actors,
+    ...directors,
+    ...tags,
+    ...region,
+    ...language,
+    year,
+    status,
+    sourceName
+  ]).join(' ');
+
+  return compactObject({
+    id,
+    title,
+    aliases,
+    description,
+    url,
+    poster,
+    year,
+    date,
+    category,
+    tags,
+    actors,
+    directors,
+    region,
+    language,
+    score,
+    status,
+    source: sourceName,
+    text,
+    raw: config.includeRaw ? item : undefined
+  });
+}
+
+export function buildDocumentUrl(item, config) {
+  const raw = firstValue(item.url, item.link, item.permalink, item.htmlUrl, item.href);
+  if (raw) return buildAssetUrl(raw, config);
+  const path = firstValue(item.path, item.pathname, item.route);
+  if (path) return buildAssetUrl(path, config);
+  const slug = firstValue(item.slug, item.id, item.vod_id);
+  if (!slug) return '';
+  return joinUrl(config.publicBase, `detail/${encodeURIComponent(String(slug))}`);
+}
+
+export function buildAssetUrl(value, config) {
+  const raw = String(value || '').trim();
+  if (!raw) return '';
+  if (/^https?:\/\//i.test(raw)) return raw;
+  if (raw.startsWith('//')) return `https:${raw}`;
+  return joinUrl(config.publicBase, raw);
+}
+
+function normalizeId(item, title, source) {
+  const explicit = firstValue(item.id, item.vod_id, item.video_id, item.uuid, item.slug, item.code, item.imdb_id, item.douban_id);
+  if (explicit) return String(explicit);
+  return `${source?.slug || 'local'}-${hashString(`${title}:${firstValue(item.url, item.link, item.path)}`)}`;
+}
+
+function normalizeYear(value) {
+  const match = String(value || '').match(/\b(19\d{2}|20\d{2})\b/);
+  return match ? Number(match[1]) : undefined;
+}
+
+function normalizeDate(value) {
+  if (!value) return '';
+  const text = String(value).trim();
+  const number = Number(text);
+  const date = Number.isFinite(number) && text.length >= 10
+    ? new Date(number > 10_000_000_000 ? number : number * 1000)
+    : new Date(text);
+  if (Number.isNaN(date.getTime())) return text;
+  return date.toISOString();
+}
+
+function normalizeNumber(value) {
+  if (value === undefined || value === null || value === '') return undefined;
+  const number = Number(value);
+  return Number.isFinite(number) ? number : undefined;
+}

package/src/renderers.js

@@ -0,0 +1,40 @@
+export function renderSearchIndex(searchIndex, options = {}) {
+  return `${JSON.stringify(searchIndex, null, options.pretty === false ? 0 : 2)}\n`;
+}
+
+export function renderSearchDocs(searchIndex, options = {}) {
+  return `${JSON.stringify(searchIndex.documents || [], null, options.pretty === false ? 0 : 2)}\n`;
+}
+
+export function renderSearchMeta(searchIndex, options = {}) {
+  const meta = {
+    version: searchIndex.version,
+    generatedAt: searchIndex.generatedAt,
+    checksum: searchIndex.checksum,
+    config: searchIndex.config,
+    stats: searchIndex.stats,
+    facets: searchIndex.facets,
+    suggestions: searchIndex.suggestions
+  };
+  return `${JSON.stringify(meta, null, options.pretty === false ? 0 : 2)}\n`;
+}
+
+export function buildSearchFiles(searchIndex, options = {}) {
+  return [
+    {
+      path: 'search-index.json',
+      contentType: 'application/json; charset=utf-8',
+      contents: renderSearchIndex(searchIndex, options)
+    },
+    {
+      path: 'search-docs.json',
+      contentType: 'application/json; charset=utf-8',
+      contents: renderSearchDocs(searchIndex, options)
+    },
+    {
+      path: 'search-meta.json',
+      contentType: 'application/json; charset=utf-8',
+      contents: renderSearchMeta(searchIndex, options)
+    }
+  ];
+}

package/src/runtime.js

@@ -0,0 +1,79 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { createConfig, describeConfig } from './config.js';
+import { extractItems, fetchSearchSources } from './client.js';
+import { buildSearchIndex } from './indexer.js';
+import { buildSearchFiles } from './renderers.js';
+
+export async function generateSearchIndex(input = {}, runtime = {}) {
+  const config = createConfig(input, input.env || runtime.env);
+  const records = [];
+  const sourceResults = [];
+
+  if (input.items) {
+    for (const item of input.items) records.push({ item, source: { label: 'local', slug: 'local', kind: 'local' } });
+  } else if (input.inputFile) {
+    const items = await readInputFile(input.inputFile);
+    for (const item of items) records.push({ item, source: { label: 'local', slug: 'local', kind: 'local' } });
+  } else {
+    const results = await fetchSearchSources(config.sources, config, runtime);
+    sourceResults.push(...results);
+    for (const result of results) {
+      for (const item of result.items) records.push({ item, source: result.source });
+    }
+  }
+
+  const index = buildSearchIndex(records, config, runtime);
+  return {
+    ok: true,
+    config: describeConfig(config),
+    sources: sourceResults.map((result) => ({
+      source: result.source.label,
+      urls: result.urls,
+      items: result.items.length
+    })),
+    index
+  };
+}
+
+export async function generateSearchBundle(input = {}, runtime = {}) {
+  const result = await generateSearchIndex(input, runtime);
+  const files = buildSearchFiles(result.index, input);
+  return {
+    ok: true,
+    config: result.config,
+    sources: result.sources,
+    index: result.index,
+    files
+  };
+}
+
+export async function writeSearchBundle(bundle, outDir) {
+  await mkdir(outDir, { recursive: true });
+  const written = [];
+  for (const file of bundle.files) {
+    const target = path.join(outDir, file.path);
+    await mkdir(path.dirname(target), { recursive: true });
+    await writeFile(target, file.contents, 'utf8');
+    written.push(target);
+  }
+  return written;
+}
+
+export async function readInputFile(filePath) {
+  const payload = JSON.parse(await readFile(filePath, 'utf8'));
+  return extractItems(payload);
+}
+
+export function buildOutputHeaders(type, searchIndexValue, config) {
+  const maxAge = config?.ttlSeconds || 600;
+  const headers = {
+    'cache-control': `public, max-age=${maxAge}`,
+    'content-type': 'application/json; charset=utf-8',
+    'x-ddys-search-index-version': searchIndexValue.version,
+    'x-ddys-search-index-documents': String(searchIndexValue.stats?.documents || 0)
+  };
+  if (searchIndexValue.checksum) headers.etag = `"${searchIndexValue.checksum}"`;
+  if (type) headers['x-ddys-search-index-type'] = type;
+  return headers;
+}

package/src/utils.js

@@ -0,0 +1,146 @@
+import { createHash } from 'node:crypto';
+
+export function normalizeBaseUrl(value, name = 'url') {
+  const raw = String(value || '').trim();
+  if (!raw) throw new Error(`${name} is required.`);
+  let url;
+  try {
+    url = new URL(raw);
+  } catch {
+    throw new Error(`${name} must be a valid URL.`);
+  }
+  if (!['http:', 'https:'].includes(url.protocol)) throw new Error(`${name} must use http or https.`);
+  return url.toString().replace(/\/+$/, '');
+}
+
+export function joinUrl(base, pathname = '') {
+  const left = String(base || '').replace(/\/+$/, '');
+  const right = String(pathname || '').replace(/^\/+/, '');
+  return right ? `${left}/${right}` : left;
+}
+
+export function splitList(value) {
+  if (Array.isArray(value)) return value.flatMap((item) => splitList(item));
+  return String(value || '')
+    .split(/[,\n;]/)
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+export function parseBoolean(value, fallback = false) {
+  if (value === undefined || value === null || value === '') return fallback;
+  if (typeof value === 'boolean') return value;
+  const normalized = String(value).trim().toLowerCase();
+  if (['1', 'true', 'yes', 'y', 'on'].includes(normalized)) return true;
+  if (['0', 'false', 'no', 'n', 'off'].includes(normalized)) return false;
+  return fallback;
+}
+
+export function parsePositiveInteger(value, fallback, name = 'value') {
+  const number = Number(value);
+  if (Number.isInteger(number) && number > 0) return number;
+  if (fallback !== undefined) return fallback;
+  throw new Error(`${name} must be a positive integer.`);
+}
+
+export function sanitizeFilePart(value, fallback = 'file') {
+  const text = String(value || '')
+    .trim()
+    .toLowerCase()
+    .replace(/https?:\/\//g, '')
+    .replace(/[^a-z0-9\u4e00-\u9fff]+/gi, '-')
+    .replace(/^-+|-+$/g, '');
+  return text || fallback;
+}
+
+export function hashString(value, length = 10) {
+  return createHash('sha1').update(String(value || '')).digest('hex').slice(0, length);
+}
+
+export function createEtag(value) {
+  return `"${hashString(typeof value === 'string' ? value : stableJson(value), 16)}"`;
+}
+
+export function stableJson(value) {
+  return JSON.stringify(value, Object.keys(flattenKeys(value)).sort());
+}
+
+function flattenKeys(value, out = {}) {
+  if (!value || typeof value !== 'object') return out;
+  for (const key of Object.keys(value)) {
+    out[key] = true;
+    flattenKeys(value[key], out);
+  }
+  return out;
+}
+
+export function uniqueStrings(values) {
+  const seen = new Set();
+  const out = [];
+  for (const value of values || []) {
+    const text = normalizeWhitespace(value);
+    if (!text) continue;
+    const key = text.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(text);
+  }
+  return out;
+}
+
+export function toArray(value) {
+  if (Array.isArray(value)) return value;
+  if (value === undefined || value === null || value === '') return [];
+  if (typeof value === 'string') return splitList(value.replace(/[|/]/g, ','));
+  return [value];
+}
+
+export function firstValue(...values) {
+  for (const value of values.flat()) {
+    if (value === undefined || value === null) continue;
+    const text = typeof value === 'string' ? normalizeWhitespace(value) : value;
+    if (text !== '') return text;
+  }
+  return '';
+}
+
+export function stripHtml(value) {
+  return String(value || '')
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/gi, ' ')
+    .replace(/&amp;/gi, '&')
+    .replace(/&lt;/gi, '<')
+    .replace(/&gt;/gi, '>')
+    .replace(/&quot;/gi, '"')
+    .replace(/&#39;/g, "'");
+}
+
+export function normalizeWhitespace(value) {
+  return String(value || '').replace(/\s+/g, ' ').trim();
+}
+
+export function createTimeoutSignal(ms) {
+  const controller = new AbortController();
+  let didTimeout = false;
+  const timer = setTimeout(() => {
+    didTimeout = true;
+    controller.abort();
+  }, ms);
+  return {
+    signal: controller.signal,
+    timedOut: () => didTimeout,
+    cancel: () => clearTimeout(timer)
+  };
+}
+
+export function compactObject(value) {
+  const out = {};
+  for (const [key, item] of Object.entries(value || {})) {
+    if (item === undefined || item === null || item === '') continue;
+    if (Array.isArray(item) && item.length === 0) continue;
+    out[key] = item;
+  }
+  return out;
+}

package/src/worker.js

@@ -0,0 +1,89 @@
+import { createConfig } from './config.js';
+import { generateSearchBundle, buildOutputHeaders } from './runtime.js';
+import { renderSearchDocs, renderSearchIndex, renderSearchMeta } from './renderers.js';
+import { searchIndex } from './indexer.js';
+
+export function createWorkerHandler(input = {}) {
+  let cache = null;
+  return async function fetch(request, env = {}, ctx = {}) {
+    return handleWorkerRequest(request, {
+      ...input,
+      env,
+      ctx,
+      cacheRef: {
+        get: () => cache,
+        set: (value) => {
+          cache = value;
+        }
+      }
+    });
+  };
+}
+
+export async function handleWorkerRequest(request, runtime = {}) {
+  const url = new URL(request.url);
+  const route = routeForPath(url.pathname);
+  const config = createConfig(runtime, runtime.env || {});
+
+  if (route === 'not-found') {
+    return jsonResponse({ ok: false, message: 'Not found' }, 404, { 'cache-control': 'no-store' });
+  }
+
+  if (route === 'health') {
+    return jsonResponse({ ok: true, service: 'ddys-search-index', version: config.version }, 200, { 'cache-control': 'no-store' });
+  }
+
+  const bundle = await getBundle(config, runtime);
+  const index = bundle.index;
+  const etag = `"${index.checksum}"`;
+  if (request.headers.get('if-none-match') === etag && route !== 'search') {
+    return new Response(null, { status: 304, headers: buildOutputHeaders(route, index, config) });
+  }
+
+  if (route === 'search') {
+    const q = url.searchParams.get('q') || '';
+    const limit = Number(url.searchParams.get('limit') || 20);
+    const result = q ? searchIndex(index, q, { limit: Number.isInteger(limit) && limit > 0 ? limit : 20 }) : { query: q, tokens: [], total: 0, items: [] };
+    return jsonResponse({ ok: true, ...result }, 200, buildOutputHeaders(route, index, config));
+  }
+
+  if (route === 'docs') return textResponse(renderSearchDocs(index), buildOutputHeaders(route, index, config));
+  if (route === 'meta') return textResponse(renderSearchMeta(index), buildOutputHeaders(route, index, config));
+  return textResponse(renderSearchIndex(index), buildOutputHeaders(route, index, config));
+}
+
+export function routeForPath(pathname) {
+  const path = String(pathname || '/').replace(/\/+$/, '') || '/';
+  if (path === '/health') return 'health';
+  if (path === '/search') return 'search';
+  if (path === '/search-index.json' || path === '/index.json' || path === '/') return 'index';
+  if (path === '/search-docs.json' || path === '/docs.json') return 'docs';
+  if (path === '/search-meta.json' || path === '/meta.json') return 'meta';
+  return 'not-found';
+}
+
+async function getBundle(config, runtime) {
+  const now = Date.now();
+  const cache = runtime.cacheRef?.get?.();
+  if (config.workerCache && cache && cache.expiresAt > now) return cache.bundle;
+  const bundle = await generateSearchBundle(config, runtime);
+  runtime.cacheRef?.set?.({
+    expiresAt: now + config.ttlSeconds * 1000,
+    bundle
+  });
+  return bundle;
+}
+
+function textResponse(body, headers) {
+  return new Response(body, { status: 200, headers });
+}
+
+function jsonResponse(body, status = 200, headers = {}) {
+  return new Response(`${JSON.stringify(body, null, 2)}\n`, {
+    status,
+    headers: {
+      'content-type': 'application/json; charset=utf-8',
+      ...headers
+    }
+  });
+}