parse-stack-next 5.5.4 → 5.5.5

data/scripts/vector_prototype/create_vector_index.js

@@ -1,105 +0,0 @@
-// Atlas $vectorSearch index for vector_prototype.WikiArticle.
-//
-// Idempotent: drops any existing search indexes on the collection before
-// creating the canonical one. Naming follows vector_rag_plan.md §3:
-//
-//   <table>_<field>_<provider>_<dimensions>_idx
-//
-// Reused by both Parse::VectorSearch tests (v4.3) and Parse::Retrieval
-// tests (v4.4). The collection's data is loaded separately by
-// fetch_embeddings.py.
-//
-// Run:
-//   mongosh "mongodb://localhost:29020/vector_prototype?directConnection=true" \
-//     scripts/vector_prototype/create_vector_index.js
-//
-// To switch to 1024-dim (voyage-multimodal-3 compat), set DIMS=1024
-// before running. Mongosh exposes shell args via passthroughs only,
-// so the value is read from an env shim or edit the constant below.
-
-// Read the manifest written by fetch_embeddings.py so dims/provider/name
-// can't drift from the loaded data. mongosh runs on Node, so use fs.
-const fs = require("fs");
-const path = require("path");
-const _scriptDir = path.dirname(typeof __filename !== "undefined" ? __filename : process.argv[1] || ".");
-const _manifestPath = path.join(_scriptDir, "fixture_manifest.json");
-const MANIFEST = JSON.parse(fs.readFileSync(_manifestPath, "utf8"));
-
-const COLL = MANIFEST.collection;
-const DIMS = MANIFEST.dims;
-const PROVIDER = MANIFEST.provider;
-const INDEX_NAME = MANIFEST.index_name;
-
-print(`[idx] target: ${db.getName()}.${COLL} → ${INDEX_NAME}`);
-
-// Drop any stale search indexes so re-runs converge to a known state.
-try {
-  const existing = db[COLL].getSearchIndexes();
-  existing.forEach(function (i) {
-    print(`  drop existing index: ${i.name}`);
-    db[COLL].dropSearchIndex(i.name);
-  });
-} catch (e) {
-  print(`  (no existing indexes / error listing: ${e.message})`);
-}
-sleep(1000);
-
-print(`[idx] creating ${INDEX_NAME} (vectorSearch, ${DIMS} dims, cosine)`);
-db[COLL].createSearchIndex(INDEX_NAME, "vectorSearch", {
-  fields: [
-    {
-      type: "vector",
-      path: "embedding",
-      numDimensions: DIMS,
-      similarity: "cosine",
-    },
-    // Filter fields — declare here anything you want to use as a
-    // $vectorSearch filter constraint. Atlas only accepts filter
-    // predicates on fields declared as type:"filter" in the index.
-    { type: "filter", path: "wiki_id" },
-  ],
-});
-
-print("[idx] waiting for queryable...");
-let attempts = 0;
-const maxAttempts = 60;
-while (attempts < maxAttempts) {
-  const found = db[COLL].getSearchIndexes().find(function (i) {
-    return i.name === INDEX_NAME;
-  });
-  if (found && found.queryable === true) {
-    print(`[idx] ready after ${attempts * 2}s`);
-    break;
-  }
-  sleep(2000);
-  attempts++;
-}
-if (attempts >= maxAttempts) {
-  print("[idx] WARNING: index not queryable yet; later queries may fail");
-}
-
-// Smoke-test: pick an arbitrary doc and find its top-5 neighbours.
-print("\n[smoke] $vectorSearch self-similarity check");
-const seed = db[COLL].findOne({});
-if (!seed) {
-  print("[smoke] no docs loaded — run fetch_embeddings.py first");
-} else {
-  const out = db[COLL].aggregate([
-    {
-      $vectorSearch: {
-        index: INDEX_NAME,
-        path: "embedding",
-        queryVector: seed.embedding,
-        numCandidates: 100,
-        limit: 5,
-      },
-    },
-    { $project: { _id: 1, title: 1, _vscore: { $meta: "vectorSearchScore" } } },
-  ]).toArray();
-  print(`  seed: ${seed.title}`);
-  out.forEach(function (r, i) {
-    print(`  ${i + 1}. score=${r._vscore.toFixed(4)}  ${r.title}`);
-  });
-}
-
-print("\n[done]");

data/scripts/vector_prototype/fetch_embeddings.py

@@ -1,241 +0,0 @@
-#!/usr/bin/env python3
-"""
-Vector-search & RAG test fixture loader.
-
-Pulls a subset of a pre-computed embeddings dataset from HuggingFace and
-loads it into Atlas Local at localhost:29020 (the same container managed
-by scripts/docker/docker-compose.atlas.yml). Designed to be reused by both:
-
-  1. Vector search integration tests (Parse::VectorSearch — v4.3 plan)
-  2. RAG retrieval tests (Parse::Retrieval — v4.4 plan)
-
-The Wikipedia article shape (title + full text + url + embedding) covers
-both surfaces: vector tests need (title, embedding), RAG/chunking tests
-need the full text body.
-
-The runtime target is voyage-multimodal-3 (1024-dim) — the production
-preference. The fixture data is provider-agnostic: any pre-computed
-embeddings exercise the same Atlas $vectorSearch surface, since the
-index contract is (path, dims, similarity) — provider is metadata.
-
-Two presets:
-
-  PRESET=fast (default) — Cohere/wikipedia-22-12-simple-embeddings
-    768-dim, ~13MB shards, ~485k rows total. Quick to download.
-    Use when iterating on pipeline mechanics, not Voyage-shape parity.
-
-  PRESET=voyage_compat — Cohere/wikipedia-2023-11-embed-multilingual-v3
-    1024-dim (matches voyage-multimodal-3), ~1.5GB shards.
-    Use when the index/query surface needs to be Voyage-dimension-shaped
-    even though vectors come from Cohere. Vectors are NOT interchangeable
-    with Voyage outputs — same dim, different latent space — but the
-    SDK pipeline mechanics are validated correctly.
-
-For actual Voyage-vector parity at test time, compute query vectors via
-the Voyage API into the same 1024-dim index (no local-inference path
-exists — Voyage models are closed-weights, API-only).
-
-Prereqs:  pip install pyarrow pymongo requests
-
-Run:      python3 scripts/vector_prototype/fetch_embeddings.py
-"""
-
-import os
-import sys
-import datetime
-import requests
-import pyarrow.parquet as pq
-from pymongo import MongoClient
-
-PRESETS = {
-    "fast": {
-        # MongoDB's reference dataset for Atlas Vector Search demos.
-        # ~3500 movies × 1536-dim OpenAI ada-002 plot embeddings.
-        # Public, ~42MB, single JSON file. Format: array of objects with
-        # plot_embedding field.
-        "url": "https://huggingface.co/datasets/MongoDB/embedded_movies/resolve/main/sample_mflix.embedded_movies.json",
-        "dims": 1536,
-        "provider": "openai-text-embedding-ada-002",
-        "format": "json",
-        "embedding_field": "plot_embedding",
-        "id_field": None,  # MongoDB will auto-assign or we synthesize
-    },
-    "voyage_compat": {
-        # 1024-dim matches voyage-multimodal-3 — same index shape, different
-        # latent space. Vectors NOT mixable with Voyage outputs at query time.
-        # NOTE: requires an authenticated HuggingFace download
-        # (HF_TOKEN env var or `huggingface-cli login`). The Cohere wikipedia
-        # v3 dataset is gated since late 2024.
-        "url": "https://huggingface.co/datasets/Cohere/wikipedia-2023-11-embed-multilingual-v3/resolve/main/en/0000.parquet",
-        "dims": 1024,
-        "provider": "cohere-embed-multilingual-v3",
-        "format": "parquet",
-        "embedding_field": "emb",
-        "id_field": "id",
-    },
-}
-
-PRESET = os.environ.get("PRESET", "fast")
-if PRESET not in PRESETS:
-    print(f"[err] unknown PRESET={PRESET}; choose one of: {list(PRESETS)}", file=sys.stderr)
-    sys.exit(2)
-
-_p = PRESETS[PRESET]
-DATASET_URL = os.environ.get("DATASET_URL", _p["url"])
-DIMS_EXPECTED = int(os.environ.get("DIMS_EXPECTED", _p["dims"]))
-PROVIDER_LABEL = os.environ.get("PROVIDER_LABEL", _p["provider"])
-DATA_FORMAT = _p["format"]
-EMBEDDING_FIELD = _p["embedding_field"]
-ID_FIELD = _p["id_field"]
-
-_ext = "parquet" if DATA_FORMAT == "parquet" else "json"
-LOCAL_FILE = os.environ.get("LOCAL_FILE", f"/tmp/parse-stack-fixture-{PRESET}.{_ext}")
-MONGO_URI = os.environ.get("ATLAS_URI", "mongodb://localhost:29020/?directConnection=true")
-DB_NAME = os.environ.get("DB_NAME", "vector_prototype")
-# Collection name mirrors the dataset shape so RAG tests can pivot
-# without coupling test assertions to a hard-coded class name.
-DEFAULT_COLL = "Movie" if PRESET == "fast" else "WikiArticle"
-COLL_NAME = os.environ.get("COLL_NAME", DEFAULT_COLL)
-LIMIT = int(os.environ.get("LIMIT", "10000"))
-
-print(f"[preset] {PRESET}  provider={PROVIDER_LABEL}  dims={DIMS_EXPECTED}")
-
-
-def download():
-    if os.path.exists(LOCAL_FILE) and os.path.getsize(LOCAL_FILE) > 0:
-        print(f"[skip] {LOCAL_FILE} already present ({os.path.getsize(LOCAL_FILE)} bytes)")
-        return
-    headers = {}
-    token = os.environ.get("HF_TOKEN")
-    if token:
-        headers["Authorization"] = f"Bearer {token}"
-    print(f"[download] {DATASET_URL}")
-    with requests.get(DATASET_URL, stream=True, timeout=120, headers=headers) as r:
-        if r.status_code == 401:
-            print(
-                f"[err] HTTP 401 — dataset requires authentication. "
-                f"Set HF_TOKEN env var (huggingface.co token) or pick a different PRESET.",
-                file=sys.stderr,
-            )
-            sys.exit(2)
-        r.raise_for_status()
-        total = int(r.headers.get("content-length", 0))
-        written = 0
-        with open(LOCAL_FILE, "wb") as f:
-            for chunk in r.iter_content(chunk_size=1024 * 1024):
-                f.write(chunk)
-                written += len(chunk)
-                if total:
-                    pct = 100 * written / total
-                    print(f"  {written // (1024*1024)}MB / {total // (1024*1024)}MB ({pct:.1f}%)", end="\r")
-        print()
-    print(f"[download] wrote {LOCAL_FILE}")
-
-
-def _read_rows():
-    if DATA_FORMAT == "parquet":
-        print(f"[read] parquet {LOCAL_FILE}")
-        table = pq.read_table(LOCAL_FILE)
-        print(f"[read] rows={table.num_rows} columns={table.column_names}")
-        return table.to_pylist()
-    elif DATA_FORMAT == "json":
-        print(f"[read] json {LOCAL_FILE}")
-        import json as _json
-        with open(LOCAL_FILE, "r") as f:
-            data = _json.load(f)
-        # embedded_movies is a top-level array
-        if not isinstance(data, list):
-            print(f"[err] expected top-level JSON array, got {type(data).__name__}", file=sys.stderr)
-            sys.exit(1)
-        print(f"[read] rows={len(data)}")
-        if data:
-            print(f"[read] sample keys: {list(data[0].keys())[:10]}")
-        return data
-    else:
-        print(f"[err] unknown DATA_FORMAT={DATA_FORMAT}", file=sys.stderr)
-        sys.exit(1)
-
-
-def load():
-    rows = _read_rows()
-    if LIMIT > 0:
-        rows = rows[:LIMIT]
-    if not rows:
-        print("[err] no rows", file=sys.stderr)
-        sys.exit(1)
-
-    # Find the first row that actually has an embedding (some datasets,
-    # including embedded_movies, have null embeddings for entries with
-    # missing plot text).
-    sample = next((r for r in rows if r.get(EMBEDDING_FIELD)), None)
-    if sample is None:
-        print(f"[err] no rows have field '{EMBEDDING_FIELD}'; sample keys: {list(rows[0].keys())}", file=sys.stderr)
-        sys.exit(1)
-    dims = len(sample[EMBEDDING_FIELD])
-    print(f"[verify] embedding field='{EMBEDDING_FIELD}'  dims={dims}")
-    if dims != DIMS_EXPECTED:
-        print(
-            f"[warn] dims={dims} differs from preset DIMS_EXPECTED={DIMS_EXPECTED}; "
-            f"manifest will record the actual dims",
-            file=sys.stderr,
-        )
-
-    now = datetime.datetime.utcnow()
-    docs = []
-    skipped = 0
-    for idx, r in enumerate(rows):
-        emb = r.get(EMBEDDING_FIELD)
-        if not emb or len(emb) != dims:
-            skipped += 1
-            continue
-        # Carry through dataset fields (text/title/etc.) so RAG/chunker
-        # tests have real content. Source fields go first; our canonical
-        # fields win.
-        doc = {k: v for k, v in r.items() if k != EMBEDDING_FIELD}
-        doc["embedding"] = list(emb)
-        doc["_created_at"] = now
-        doc["_updated_at"] = now
-        if ID_FIELD and r.get(ID_FIELD) is not None:
-            doc["_id"] = f"{COLL_NAME.lower()}_{r[ID_FIELD]}"
-        else:
-            doc["_id"] = f"{COLL_NAME.lower()}_{idx:06d}"
-        docs.append(doc)
-
-    if skipped:
-        print(f"[load] skipped {skipped} rows missing/short embeddings")
-
-    client = MongoClient(MONGO_URI)
-    coll = client[DB_NAME][COLL_NAME]
-    print(f"[mongo] dropping {DB_NAME}.{COLL_NAME}")
-    coll.drop()
-
-    # Bulk insert in chunks — pymongo's default is fine but explicit is clearer
-    BATCH = 1000
-    for i in range(0, len(docs), BATCH):
-        coll.insert_many(docs[i:i + BATCH], ordered=False)
-        print(f"  inserted {min(i + BATCH, len(docs))}/{len(docs)}")
-
-    count = coll.count_documents({})
-    print(f"[mongo] {DB_NAME}.{COLL_NAME} now has {count} docs (embedding dims={dims})")
-
-    # Manifest — single source of truth shared with create_vector_index.js
-    # so the index name + dimensions can never drift from the loaded data.
-    import json
-    manifest = {
-        "preset": PRESET,
-        "provider": PROVIDER_LABEL,
-        "dims": dims,
-        "db": DB_NAME,
-        "collection": COLL_NAME,
-        "count": count,
-        "index_name": f"{COLL_NAME}_embedding_{PROVIDER_LABEL.replace('-', '_')}_{dims}_idx",
-    }
-    manifest_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixture_manifest.json")
-    with open(manifest_path, "w") as f:
-        json.dump(manifest, f, indent=2)
-    print(f"[manifest] wrote {manifest_path}: {manifest['index_name']}")
-
-
-if __name__ == "__main__":
-    download()
-    load()

data/scripts/vector_prototype/fixture_manifest.json

@@ -1,9 +0,0 @@
-{
-  "preset": "fast",
-  "provider": "openai-text-embedding-ada-002",
-  "dims": 1536,
-  "db": "vector_prototype",
-  "collection": "Movie",
-  "count": 1472,
-  "index_name": "Movie_embedding_openai_text_embedding_ada_002_1536_idx"
-}

data/scripts/vector_prototype/query_prototype.rb

@@ -1,84 +0,0 @@
-#!/usr/bin/env ruby
-# Exercise Atlas $vectorSearch end-to-end via the Ruby mongo driver — the
-# same driver Parse::MongoDB.aggregate uses. This is the literal pipeline
-# shape Parse::VectorSearch will produce (vector_rag_plan.md §3).
-#
-# Run:
-#   bundle exec ruby scripts/vector_prototype/query_prototype.rb
-#
-# Prereq: fetch_embeddings.py + create_vector_index.js have been run, so
-# vector_prototype.WikiArticle is populated and the search index is
-# queryable.
-#
-# This script intentionally avoids any Parse::* SDK code — it's the raw
-# Atlas surface, used as the ground-truth comparison once Parse::VectorSearch
-# lands. The same pipeline shape will be emitted by Parse::VectorSearch::SearchBuilder.
-
-require "mongo"
-
-require "json"
-
-MANIFEST_PATH = File.expand_path("fixture_manifest.json", __dir__)
-unless File.exist?(MANIFEST_PATH)
-  abort "no fixture_manifest.json — run fetch_embeddings.py first"
-end
-MANIFEST = JSON.parse(File.read(MANIFEST_PATH))
-
-MONGO_URI = ENV.fetch("ATLAS_URI", "mongodb://localhost:29020/#{MANIFEST['db']}?directConnection=true")
-INDEX_NAME = ENV.fetch("VECTOR_INDEX", MANIFEST["index_name"])
-COLL_NAME = MANIFEST["collection"].to_sym
-
-puts "[manifest] preset=#{MANIFEST['preset']}  provider=#{MANIFEST['provider']}  dims=#{MANIFEST['dims']}  index=#{INDEX_NAME}"
-
-client = Mongo::Client.new(MONGO_URI)
-coll = client[COLL_NAME]
-
-count = coll.count_documents({})
-abort "no docs loaded — run fetch_embeddings.py first" if count.zero?
-puts "[setup] #{count} docs in #{client.database.name}.#{COLL_NAME}"
-
-# Use an existing doc's vector as the query — exercises the index without
-# requiring an embedding API. When Voyage lands, swap this for a freshly
-# computed query vector against the same index.
-seed = coll.find.limit(1).first
-puts "[seed] #{seed["title"]}  (#{seed["embedding"].size}-dim)"
-
-pipeline = [
-  {
-    "$vectorSearch" => {
-      "index"         => INDEX_NAME,
-      "path"          => "embedding",
-      "queryVector"   => seed["embedding"],
-      "numCandidates" => 200,
-      "limit"         => 10,
-    },
-  },
-  {
-    "$project" => {
-      "_id"     => 1,
-      "title"   => 1,
-      # Project the score under _vscore (not _score) so hybrid search with
-      # Atlas Search lexical scores doesn't collide. Matches the convention
-      # the SDK will adopt — vector_rag_plan.md §3.
-      "_vscore" => { "$meta" => "vectorSearchScore" },
-    },
-  },
-]
-
-puts "[query] $vectorSearch  limit=10  numCandidates=200"
-t0 = Time.now
-results = coll.aggregate(pipeline).to_a
-elapsed_ms = ((Time.now - t0) * 1000).round(1)
-
-puts "[result] #{results.size} hits in #{elapsed_ms}ms"
-results.each_with_index do |r, i|
-  printf("  %2d. score=%.4f  %s\n", i + 1, r["_vscore"], r["title"])
-end
-
-# Sanity: top hit should be the seed itself (cosine = 1.0)
-top = results.first
-if top && top["_id"] == seed["_id"]
-  puts "[ok] top hit == seed (self-similarity verified)"
-else
-  puts "[warn] top hit was not the seed — index may still be building"
-end

data/scripts/vector_prototype/run.sh

@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-# Orchestrate the vector-search / RAG test fixture setup.
-# Assumes the Atlas Local container from docker-compose.atlas.yml is up
-# (i.e. localhost:29020 is reachable).
-#
-# Usage:  ./scripts/vector_prototype/run.sh
-#
-# Once this stabilises, the fetch + index-create steps will be folded
-# into scripts/docker/docker-compose.atlas.yml as additional init
-# containers, alongside the existing atlas-init service.
-
-set -euo pipefail
-
-HERE="$(cd "$(dirname "$0")" && pwd)"
-
-echo "[1/3] verifying Atlas Local on localhost:29020"
-if ! mongosh --quiet --eval "db.runCommand({ ping: 1 })" \
-     "mongodb://localhost:29020/?directConnection=true" >/dev/null; then
-  echo "  ERROR: Atlas Local not reachable. Start it with:"
-  echo "    docker-compose -f scripts/docker/docker-compose.atlas.yml up -d"
-  exit 1
-fi
-echo "  ok"
-
-echo "[2/3] downloading + loading embeddings"
-python3 "$HERE/fetch_embeddings.py"
-
-echo "[3/3] creating vectorSearch index"
-mongosh --quiet "mongodb://localhost:29020/vector_prototype?directConnection=true" \
-  "$HERE/create_vector_index.js"
-
-echo
-echo "Done. Run the Ruby query exercise with:"
-echo "  bundle exec ruby scripts/vector_prototype/query_prototype.rb"