@mixpeek/react-searchkit 0.1.2

package/scripts/index.mjs

@@ -0,0 +1,364 @@
+#!/usr/bin/env node
+/**
+ * mixpeek-index — Scrape a website and index it into a Mixpeek collection,
+ * then scaffold a retriever ready for use with <SearchKit />.
+ *
+ * Usage:
+ *   mixpeek-index --api-key <KEY> --url <URL> [options]
+ *
+ * Options:
+ *   --api-key       Mixpeek API key (required)
+ *   --url           Website URL to scrape and index (required)
+ *   --base-url      Mixpeek API base URL (default: https://api.mixpeek.com)
+ *   --namespace     Namespace ID to use (created if --namespace-name is set)
+ *   --namespace-name  Name for a new namespace (creates one if --namespace not given)
+ *   --slug          Retriever slug for SearchKit (default: searchkit-<timestamp>)
+ *   --wait          Wait for indexing to complete before exiting (default: true)
+ *   --poll-interval Seconds between batch status polls (default: 10)
+ *   --timeout       Max seconds to wait for indexing (default: 300)
+ *   --help          Show this help
+ *
+ * Example:
+ *   mixpeek-index --api-key mxp_sk_abc123 --url https://docs.example.com
+ */
+
+import { parseArgs } from "node:util";
+
+const USAGE = `
+mixpeek-index — Scrape a website and index it into Mixpeek, then scaffold a retriever.
+
+Usage:
+  mixpeek-index --api-key <KEY> --url <URL> [options]
+
+Options:
+  --api-key          Mixpeek API key (required)
+  --url              Website URL to scrape and index (required)
+  --base-url         API base URL (default: https://api.mixpeek.com)
+  --namespace        Existing namespace ID to use
+  --namespace-name   Name for a new namespace (auto-created if --namespace not set)
+  --slug             Retriever name/slug for SearchKit (default: searchkit-<timestamp>)
+  --wait             Wait for indexing to complete (default: true)
+  --poll-interval    Poll interval in seconds (default: 10)
+  --timeout          Max wait time in seconds (default: 300)
+  --help             Show this help
+
+Example:
+  mixpeek-index --api-key mxp_sk_abc123 --url https://docs.example.com --slug my-docs-search
+`;
+
+let args;
+try {
+  args = parseArgs({
+    options: {
+      "api-key": { type: "string" },
+      url: { type: "string" },
+      "base-url": { type: "string", default: "https://api.mixpeek.com" },
+      namespace: { type: "string" },
+      "namespace-name": { type: "string" },
+      slug: { type: "string" },
+      wait: { type: "boolean", default: true },
+      "poll-interval": { type: "string", default: "10" },
+      timeout: { type: "string", default: "300" },
+      help: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+} catch (e) {
+  console.error(`Error: ${e.message}\n`);
+  console.error(USAGE);
+  process.exit(1);
+}
+
+if (args.values.help) {
+  console.log(USAGE);
+  process.exit(0);
+}
+
+const apiKey = args.values["api-key"];
+const targetUrl = args.values["url"];
+
+if (!apiKey) {
+  console.error("Error: --api-key is required.\n");
+  process.exit(1);
+}
+if (!targetUrl) {
+  console.error("Error: --url is required.\n");
+  process.exit(1);
+}
+
+const BASE_URL = (args.values["base-url"] || "https://api.mixpeek.com").replace(/\/+$/, "");
+const slug = args.values["slug"] || `searchkit-${Date.now()}`;
+const shouldWait = args.values["wait"];
+const pollInterval = parseInt(args.values["poll-interval"] || "10", 10) * 1000;
+const timeout = parseInt(args.values["timeout"] || "300", 10) * 1000;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function headers(nsId) {
+  const h = {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${apiKey}`,
+  };
+  if (nsId) h["X-Namespace"] = nsId;
+  return h;
+}
+
+async function api(method, path, body, nsId) {
+  const res = await fetch(`${BASE_URL}${path}`, {
+    method,
+    headers: headers(nsId),
+    body: body ? JSON.stringify(body) : undefined,
+  });
+  const data = await res.json();
+  if (!res.ok) {
+    const msg = data?.error?.message || data?.detail || JSON.stringify(data);
+    throw new Error(`${method} ${path} → ${res.status}: ${msg}`);
+  }
+  return data;
+}
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+// ---------------------------------------------------------------------------
+// Main flow
+// ---------------------------------------------------------------------------
+
+async function run() {
+  console.log(`\n🌐 mixpeek-index — Indexing ${targetUrl}\n`);
+
+  // ── Step 1: Resolve namespace ──────────────────────────────────────────────
+  let namespaceId = args.values["namespace"];
+
+  if (!namespaceId) {
+    const nsName = args.values["namespace-name"] || `searchkit-ns-${Date.now()}`;
+    console.log(`📦 Creating namespace "${nsName}"...`);
+    const nsData = await api("POST", "/v1/namespaces", {
+      namespace_name: nsName,
+      feature_extractors: [
+        { feature_extractor_name: "web_scraper", version: "v1" },
+        { feature_extractor_name: "text_extractor", version: "v1" },
+      ],
+    });
+    namespaceId = nsData.namespace_id;
+    console.log(`   ✓ Namespace: ${namespaceId}`);
+  } else {
+    console.log(`   ✓ Using existing namespace: ${namespaceId}`);
+  }
+
+  // ── Step 2: Create bucket ──────────────────────────────────────────────────
+  const bucketName = `${slug}-bucket`;
+  console.log(`\n🗄️  Creating bucket "${bucketName}"...`);
+  const bucketData = await api(
+    "POST",
+    "/v1/buckets",
+    {
+      bucket_name: bucketName,
+      description: `Web content bucket for ${targetUrl}`,
+      bucket_schema: {
+        properties: {
+          url: { type: "string" },
+          title: { type: "string" },
+        },
+      },
+    },
+    namespaceId
+  );
+  const bucketId = bucketData.bucket_id;
+  console.log(`   ✓ Bucket: ${bucketId}`);
+
+  // ── Step 3: Add URL as bucket object ──────────────────────────────────────
+  console.log(`\n📎 Adding ${targetUrl} as bucket object...`);
+  const objData = await api(
+    "POST",
+    `/v1/buckets/${bucketId}/objects`,
+    {
+      url: targetUrl,
+      metadata: { title: new URL(targetUrl).hostname, source: "web-scrape" },
+    },
+    namespaceId
+  );
+  const objectId = objData.object_id;
+  console.log(`   ✓ Object: ${objectId}`);
+
+  // ── Step 4: Create collection with web_scraper ─────────────────────────────
+  const collectionName = `${slug}-collection`;
+  console.log(`\n🗂️  Creating collection "${collectionName}"...`);
+  const colData = await api(
+    "POST",
+    "/v1/collections",
+    {
+      collection_name: collectionName,
+      description: `Web-scraped content from ${targetUrl}`,
+      source: { type: "bucket", bucket_ids: [bucketId] },
+      feature_extractor: {
+        feature_extractor_name: "web_scraper",
+        version: "v1",
+        input_mappings: { url: "url" },
+      },
+    },
+    namespaceId
+  );
+  const collectionId = colData.collection_id;
+  console.log(`   ✓ Collection: ${collectionId}`);
+
+  // ── Step 5: Trigger processing ────────────────────────────────────────────
+  console.log(`\n⚙️  Triggering collection processing...`);
+  const triggerData = await api(
+    "POST",
+    `/v1/collections/${collectionId}/trigger`,
+    {},
+    namespaceId
+  );
+  const batchId = triggerData.batch_id;
+  console.log(`   ✓ Batch: ${batchId} (${triggerData.object_count} objects)`);
+
+  // ── Step 6: Wait for completion ───────────────────────────────────────────
+  if (shouldWait) {
+    console.log(`\n⏳ Waiting for indexing to complete (timeout: ${timeout / 1000}s)...`);
+    const deadline = Date.now() + timeout;
+    let dotCount = 0;
+
+    while (Date.now() < deadline) {
+      await sleep(pollInterval);
+      const batchInfo = await api(
+        "GET",
+        `/v1/buckets/${bucketId}/batches/${batchId}`,
+        null,
+        namespaceId
+      );
+      const status = batchInfo.status || "UNKNOWN";
+      process.stdout.write(`\r   ${".".repeat((dotCount++ % 3) + 1).padEnd(3)} ${status}        `);
+
+      if (status === "COMPLETED") {
+        const docCount = batchInfo.document_count || "?";
+        console.log(`\n   ✓ Indexing complete! Documents: ${docCount}`);
+        break;
+      }
+      if (status === "FAILED" || status === "ERROR") {
+        console.error(`\n   ✗ Batch failed with status: ${status}`);
+        process.exit(1);
+      }
+    }
+  }
+
+  // ── Step 7: Get feature URI ───────────────────────────────────────────────
+  console.log(`\n🔍 Discovering collection features...`);
+  let featureUri = `mixpeek://web_scraper@v1/intfloat__multilingual_e5_large_instruct`;
+  try {
+    const featData = await api(
+      "GET",
+      `/v1/collections/${collectionId}/features`,
+      null,
+      namespaceId
+    );
+    const features = featData.features || [];
+    const textFeature = features.find(
+      (f) => f.vector_index?.purpose === "text" || f.feature_address?.includes("multilingual")
+    );
+    if (textFeature?.feature_address) {
+      featureUri = textFeature.feature_address;
+    }
+  } catch {
+    console.log(`   ℹ️  Using default feature URI`);
+  }
+  console.log(`   ✓ Feature URI: ${featureUri}`);
+
+  // ── Step 8: Create retriever ──────────────────────────────────────────────
+  console.log(`\n🔎 Creating retriever "${slug}"...`);
+  const retData = await api(
+    "POST",
+    "/v1/retrievers",
+    {
+      retriever_name: slug,
+      description: `SearchKit retriever for ${targetUrl}`,
+      collection_identifiers: [collectionId],
+      input_schema: {
+        query: { type: "text", required: true, description: "Search query" },
+        doc_type: { type: "text", required: false, description: "Filter by page type (page/code/image)" },
+        keyword: { type: "text", required: false, description: "Keyword filter on content" },
+      },
+      stages: [
+        {
+          stage_name: "semantic_search",
+          stage_type: "filter",
+          config: {
+            stage_id: "feature_search",
+            parameters: {
+              searches: [
+                {
+                  feature_uri: featureUri,
+                  query: { input_mode: "text", value: "{{INPUT.query}}" },
+                  top_k: 20,
+                },
+              ],
+              final_top_k: 20,
+            },
+          },
+        },
+        {
+          stage_name: "sort_by_relevance",
+          stage_type: "sort",
+          config: {
+            stage_id: "sort_relevance",
+            parameters: { score_field: "score", direction: "desc" },
+          },
+        },
+      ],
+    },
+    namespaceId
+  );
+  const retrieverId = retData.retriever?.retriever_id || retData.retriever_id;
+  console.log(`   ✓ Retriever: ${retrieverId}`);
+
+  // ── Done ──────────────────────────────────────────────────────────────────
+  console.log(`
+╔════════════════════════════════════════════════════════╗
+║  ✅  Indexing complete!                                ║
+╚════════════════════════════════════════════════════════╝
+
+Resources created:
+  namespace_id:   ${namespaceId}
+  bucket_id:      ${bucketId}
+  collection_id:  ${collectionId}
+  retriever_id:   ${retrieverId}
+
+Use with SearchKit (React component):
+
+  import { SearchKit } from "@mixpeek/react-searchkit";
+
+  <SearchKit
+    projectKey="${retrieverId}"
+    bearerToken="<YOUR_API_KEY>"
+    apiBaseUrl="${BASE_URL}"
+    placeholder="Search ${new URL(targetUrl).hostname}..."
+  />
+
+Or test via API:
+
+  curl -X POST ${BASE_URL}/v1/retrievers/${retrieverId}/execute \\
+    -H "Authorization: Bearer ${apiKey}" \\
+    -H "X-Namespace: ${namespaceId}" \\
+    -H "Content-Type: application/json" \\
+    -d '{"inputs": {"query": "your search query"}, "settings": {"limit": 5}}'
+`);
+
+  // Output machine-readable JSON for piping/scripting
+  const output = {
+    namespace_id: namespaceId,
+    bucket_id: bucketId,
+    collection_id: collectionId,
+    retriever_id: retrieverId,
+    api_key: apiKey,
+    base_url: BASE_URL,
+  };
+  process.stdout.write("\n__RESULT__=" + JSON.stringify(output) + "\n");
+}
+
+run().catch((err) => {
+  console.error(`\n❌ Error: ${err.message}`);
+  process.exit(1);
+});