parse-stack-next 4.5.0 → 5.0.0

data/scripts/vector_prototype/fetch_embeddings.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+"""
+Vector-search & RAG test fixture loader.
+
+Pulls a subset of a pre-computed embeddings dataset from HuggingFace and
+loads it into Atlas Local at localhost:27020 (the same container managed
+by scripts/docker/docker-compose.atlas.yml). Designed to be reused by both:
+
+  1. Vector search integration tests (Parse::VectorSearch — v4.3 plan)
+  2. RAG retrieval tests (Parse::Retrieval — v4.4 plan)
+
+The Wikipedia article shape (title + full text + url + embedding) covers
+both surfaces: vector tests need (title, embedding), RAG/chunking tests
+need the full text body.
+
+The runtime target is voyage-multimodal-3 (1024-dim) — the production
+preference. The fixture data is provider-agnostic: any pre-computed
+embeddings exercise the same Atlas $vectorSearch surface, since the
+index contract is (path, dims, similarity) — provider is metadata.
+
+Two presets:
+
+  PRESET=fast (default) — Cohere/wikipedia-22-12-simple-embeddings
+    768-dim, ~13MB shards, ~485k rows total. Quick to download.
+    Use when iterating on pipeline mechanics, not Voyage-shape parity.
+
+  PRESET=voyage_compat — Cohere/wikipedia-2023-11-embed-multilingual-v3
+    1024-dim (matches voyage-multimodal-3), ~1.5GB shards.
+    Use when the index/query surface needs to be Voyage-dimension-shaped
+    even though vectors come from Cohere. Vectors are NOT interchangeable
+    with Voyage outputs — same dim, different latent space — but the
+    SDK pipeline mechanics are validated correctly.
+
+For actual Voyage-vector parity at test time, compute query vectors via
+the Voyage API into the same 1024-dim index (no local-inference path
+exists — Voyage models are closed-weights, API-only).
+
+Prereqs:  pip install pyarrow pymongo requests
+
+Run:      python3 scripts/vector_prototype/fetch_embeddings.py
+"""
+
+import os
+import sys
+import datetime
+import requests
+import pyarrow.parquet as pq
+from pymongo import MongoClient
+
+PRESETS = {
+    "fast": {
+        # MongoDB's reference dataset for Atlas Vector Search demos.
+        # ~3500 movies × 1536-dim OpenAI ada-002 plot embeddings.
+        # Public, ~42MB, single JSON file. Format: array of objects with
+        # plot_embedding field.
+        "url": "https://huggingface.co/datasets/MongoDB/embedded_movies/resolve/main/sample_mflix.embedded_movies.json",
+        "dims": 1536,
+        "provider": "openai-text-embedding-ada-002",
+        "format": "json",
+        "embedding_field": "plot_embedding",
+        "id_field": None,  # MongoDB will auto-assign or we synthesize
+    },
+    "voyage_compat": {
+        # 1024-dim matches voyage-multimodal-3 — same index shape, different
+        # latent space. Vectors NOT mixable with Voyage outputs at query time.
+        # NOTE: requires an authenticated HuggingFace download
+        # (HF_TOKEN env var or `huggingface-cli login`). The Cohere wikipedia
+        # v3 dataset is gated since late 2024.
+        "url": "https://huggingface.co/datasets/Cohere/wikipedia-2023-11-embed-multilingual-v3/resolve/main/en/0000.parquet",
+        "dims": 1024,
+        "provider": "cohere-embed-multilingual-v3",
+        "format": "parquet",
+        "embedding_field": "emb",
+        "id_field": "id",
+    },
+}
+
+PRESET = os.environ.get("PRESET", "fast")
+if PRESET not in PRESETS:
+    print(f"[err] unknown PRESET={PRESET}; choose one of: {list(PRESETS)}", file=sys.stderr)
+    sys.exit(2)
+
+_p = PRESETS[PRESET]
+DATASET_URL = os.environ.get("DATASET_URL", _p["url"])
+DIMS_EXPECTED = int(os.environ.get("DIMS_EXPECTED", _p["dims"]))
+PROVIDER_LABEL = os.environ.get("PROVIDER_LABEL", _p["provider"])
+DATA_FORMAT = _p["format"]
+EMBEDDING_FIELD = _p["embedding_field"]
+ID_FIELD = _p["id_field"]
+
+_ext = "parquet" if DATA_FORMAT == "parquet" else "json"
+LOCAL_FILE = os.environ.get("LOCAL_FILE", f"/tmp/parse-stack-fixture-{PRESET}.{_ext}")
+MONGO_URI = os.environ.get("ATLAS_URI", "mongodb://localhost:27020/?directConnection=true")
+DB_NAME = os.environ.get("DB_NAME", "vector_prototype")
+# Collection name mirrors the dataset shape so RAG tests can pivot
+# without coupling test assertions to a hard-coded class name.
+DEFAULT_COLL = "Movie" if PRESET == "fast" else "WikiArticle"
+COLL_NAME = os.environ.get("COLL_NAME", DEFAULT_COLL)
+LIMIT = int(os.environ.get("LIMIT", "10000"))
+
+print(f"[preset] {PRESET}  provider={PROVIDER_LABEL}  dims={DIMS_EXPECTED}")
+
+
+def download():
+    if os.path.exists(LOCAL_FILE) and os.path.getsize(LOCAL_FILE) > 0:
+        print(f"[skip] {LOCAL_FILE} already present ({os.path.getsize(LOCAL_FILE)} bytes)")
+        return
+    headers = {}
+    token = os.environ.get("HF_TOKEN")
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    print(f"[download] {DATASET_URL}")
+    with requests.get(DATASET_URL, stream=True, timeout=120, headers=headers) as r:
+        if r.status_code == 401:
+            print(
+                f"[err] HTTP 401 — dataset requires authentication. "
+                f"Set HF_TOKEN env var (huggingface.co token) or pick a different PRESET.",
+                file=sys.stderr,
+            )
+            sys.exit(2)
+        r.raise_for_status()
+        total = int(r.headers.get("content-length", 0))
+        written = 0
+        with open(LOCAL_FILE, "wb") as f:
+            for chunk in r.iter_content(chunk_size=1024 * 1024):
+                f.write(chunk)
+                written += len(chunk)
+                if total:
+                    pct = 100 * written / total
+                    print(f"  {written // (1024*1024)}MB / {total // (1024*1024)}MB ({pct:.1f}%)", end="\r")
+        print()
+    print(f"[download] wrote {LOCAL_FILE}")
+
+
+def _read_rows():
+    if DATA_FORMAT == "parquet":
+        print(f"[read] parquet {LOCAL_FILE}")
+        table = pq.read_table(LOCAL_FILE)
+        print(f"[read] rows={table.num_rows} columns={table.column_names}")
+        return table.to_pylist()
+    elif DATA_FORMAT == "json":
+        print(f"[read] json {LOCAL_FILE}")
+        import json as _json
+        with open(LOCAL_FILE, "r") as f:
+            data = _json.load(f)
+        # embedded_movies is a top-level array
+        if not isinstance(data, list):
+            print(f"[err] expected top-level JSON array, got {type(data).__name__}", file=sys.stderr)
+            sys.exit(1)
+        print(f"[read] rows={len(data)}")
+        if data:
+            print(f"[read] sample keys: {list(data[0].keys())[:10]}")
+        return data
+    else:
+        print(f"[err] unknown DATA_FORMAT={DATA_FORMAT}", file=sys.stderr)
+        sys.exit(1)
+
+
+def load():
+    rows = _read_rows()
+    if LIMIT > 0:
+        rows = rows[:LIMIT]
+    if not rows:
+        print("[err] no rows", file=sys.stderr)
+        sys.exit(1)
+
+    # Find the first row that actually has an embedding (some datasets,
+    # including embedded_movies, have null embeddings for entries with
+    # missing plot text).
+    sample = next((r for r in rows if r.get(EMBEDDING_FIELD)), None)
+    if sample is None:
+        print(f"[err] no rows have field '{EMBEDDING_FIELD}'; sample keys: {list(rows[0].keys())}", file=sys.stderr)
+        sys.exit(1)
+    dims = len(sample[EMBEDDING_FIELD])
+    print(f"[verify] embedding field='{EMBEDDING_FIELD}'  dims={dims}")
+    if dims != DIMS_EXPECTED:
+        print(
+            f"[warn] dims={dims} differs from preset DIMS_EXPECTED={DIMS_EXPECTED}; "
+            f"manifest will record the actual dims",
+            file=sys.stderr,
+        )
+
+    now = datetime.datetime.utcnow()
+    docs = []
+    skipped = 0
+    for idx, r in enumerate(rows):
+        emb = r.get(EMBEDDING_FIELD)
+        if not emb or len(emb) != dims:
+            skipped += 1
+            continue
+        # Carry through dataset fields (text/title/etc.) so RAG/chunker
+        # tests have real content. Source fields go first; our canonical
+        # fields win.
+        doc = {k: v for k, v in r.items() if k != EMBEDDING_FIELD}
+        doc["embedding"] = list(emb)
+        doc["_created_at"] = now
+        doc["_updated_at"] = now
+        if ID_FIELD and r.get(ID_FIELD) is not None:
+            doc["_id"] = f"{COLL_NAME.lower()}_{r[ID_FIELD]}"
+        else:
+            doc["_id"] = f"{COLL_NAME.lower()}_{idx:06d}"
+        docs.append(doc)
+
+    if skipped:
+        print(f"[load] skipped {skipped} rows missing/short embeddings")
+
+    client = MongoClient(MONGO_URI)
+    coll = client[DB_NAME][COLL_NAME]
+    print(f"[mongo] dropping {DB_NAME}.{COLL_NAME}")
+    coll.drop()
+
+    # Bulk insert in chunks — pymongo's default is fine but explicit is clearer
+    BATCH = 1000
+    for i in range(0, len(docs), BATCH):
+        coll.insert_many(docs[i:i + BATCH], ordered=False)
+        print(f"  inserted {min(i + BATCH, len(docs))}/{len(docs)}")
+
+    count = coll.count_documents({})
+    print(f"[mongo] {DB_NAME}.{COLL_NAME} now has {count} docs (embedding dims={dims})")
+
+    # Manifest — single source of truth shared with create_vector_index.js
+    # so the index name + dimensions can never drift from the loaded data.
+    import json
+    manifest = {
+        "preset": PRESET,
+        "provider": PROVIDER_LABEL,
+        "dims": dims,
+        "db": DB_NAME,
+        "collection": COLL_NAME,
+        "count": count,
+        "index_name": f"{COLL_NAME}_embedding_{PROVIDER_LABEL.replace('-', '_')}_{dims}_idx",
+    }
+    manifest_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixture_manifest.json")
+    with open(manifest_path, "w") as f:
+        json.dump(manifest, f, indent=2)
+    print(f"[manifest] wrote {manifest_path}: {manifest['index_name']}")
+
+
+if __name__ == "__main__":
+    download()
+    load()

data/scripts/vector_prototype/fixture_manifest.json

@@ -0,0 +1,9 @@
+{
+  "preset": "fast",
+  "provider": "openai-text-embedding-ada-002",
+  "dims": 1536,
+  "db": "vector_prototype",
+  "collection": "Movie",
+  "count": 1472,
+  "index_name": "Movie_embedding_openai_text_embedding_ada_002_1536_idx"
+}

data/scripts/vector_prototype/query_prototype.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+# Exercise Atlas $vectorSearch end-to-end via the Ruby mongo driver — the
+# same driver Parse::MongoDB.aggregate uses. This is the literal pipeline
+# shape Parse::VectorSearch will produce (vector_rag_plan.md §3).
+#
+# Run:
+#   bundle exec ruby scripts/vector_prototype/query_prototype.rb
+#
+# Prereq: fetch_embeddings.py + create_vector_index.js have been run, so
+# vector_prototype.WikiArticle is populated and the search index is
+# queryable.
+#
+# This script intentionally avoids any Parse::* SDK code — it's the raw
+# Atlas surface, used as the ground-truth comparison once Parse::VectorSearch
+# lands. The same pipeline shape will be emitted by Parse::VectorSearch::SearchBuilder.
+
+require "mongo"
+
+require "json"
+
+MANIFEST_PATH = File.expand_path("fixture_manifest.json", __dir__)
+unless File.exist?(MANIFEST_PATH)
+  abort "no fixture_manifest.json — run fetch_embeddings.py first"
+end
+MANIFEST = JSON.parse(File.read(MANIFEST_PATH))
+
+MONGO_URI = ENV.fetch("ATLAS_URI", "mongodb://localhost:27020/#{MANIFEST['db']}?directConnection=true")
+INDEX_NAME = ENV.fetch("VECTOR_INDEX", MANIFEST["index_name"])
+COLL_NAME = MANIFEST["collection"].to_sym
+
+puts "[manifest] preset=#{MANIFEST['preset']}  provider=#{MANIFEST['provider']}  dims=#{MANIFEST['dims']}  index=#{INDEX_NAME}"
+
+client = Mongo::Client.new(MONGO_URI)
+coll = client[COLL_NAME]
+
+count = coll.count_documents({})
+abort "no docs loaded — run fetch_embeddings.py first" if count.zero?
+puts "[setup] #{count} docs in #{client.database.name}.#{COLL_NAME}"
+
+# Use an existing doc's vector as the query — exercises the index without
+# requiring an embedding API. When Voyage lands, swap this for a freshly
+# computed query vector against the same index.
+seed = coll.find.limit(1).first
+puts "[seed] #{seed["title"]}  (#{seed["embedding"].size}-dim)"
+
+pipeline = [
+  {
+    "$vectorSearch" => {
+      "index"         => INDEX_NAME,
+      "path"          => "embedding",
+      "queryVector"   => seed["embedding"],
+      "numCandidates" => 200,
+      "limit"         => 10,
+    },
+  },
+  {
+    "$project" => {
+      "_id"     => 1,
+      "title"   => 1,
+      # Project the score under _vscore (not _score) so hybrid search with
+      # Atlas Search lexical scores doesn't collide. Matches the convention
+      # the SDK will adopt — vector_rag_plan.md §3.
+      "_vscore" => { "$meta" => "vectorSearchScore" },
+    },
+  },
+]
+
+puts "[query] $vectorSearch  limit=10  numCandidates=200"
+t0 = Time.now
+results = coll.aggregate(pipeline).to_a
+elapsed_ms = ((Time.now - t0) * 1000).round(1)
+
+puts "[result] #{results.size} hits in #{elapsed_ms}ms"
+results.each_with_index do |r, i|
+  printf("  %2d. score=%.4f  %s\n", i + 1, r["_vscore"], r["title"])
+end
+
+# Sanity: top hit should be the seed itself (cosine = 1.0)
+top = results.first
+if top && top["_id"] == seed["_id"]
+  puts "[ok] top hit == seed (self-similarity verified)"
+else
+  puts "[warn] top hit was not the seed — index may still be building"
+end

data/scripts/vector_prototype/run.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Orchestrate the vector-search / RAG test fixture setup.
+# Assumes the Atlas Local container from docker-compose.atlas.yml is up
+# (i.e. localhost:27020 is reachable).
+#
+# Usage:  ./scripts/vector_prototype/run.sh
+#
+# Once this stabilises, the fetch + index-create steps will be folded
+# into scripts/docker/docker-compose.atlas.yml as additional init
+# containers, alongside the existing atlas-init service.
+
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+
+echo "[1/3] verifying Atlas Local on localhost:27020"
+if ! mongosh --quiet --eval "db.runCommand({ ping: 1 })" \
+     "mongodb://localhost:27020/?directConnection=true" >/dev/null; then
+  echo "  ERROR: Atlas Local not reachable. Start it with:"
+  echo "    docker-compose -f scripts/docker/docker-compose.atlas.yml up -d"
+  exit 1
+fi
+echo "  ok"
+
+echo "[2/3] downloading + loading embeddings"
+python3 "$HERE/fetch_embeddings.py"
+
+echo "[3/3] creating vectorSearch index"
+mongosh --quiet "mongodb://localhost:27020/vector_prototype?directConnection=true" \
+  "$HERE/create_vector_index.js"
+
+echo
+echo "Done. Run the Ruby query exercise with:"
+echo "  bundle exec ruby scripts/vector_prototype/query_prototype.rb"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parse-stack-next
 version: !ruby/object:Gem::Version
-  version: 4.5.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Anthony Persaud
@@ -113,6 +113,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -161,9 +181,25 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-description: Parse Server Ruby Client. Perform Object-relational mapping between Parse
-  Server and Ruby classes, with authentication, cloud code webhooks, push notifications
-  and more built in.
+- !ruby/object:Gem::Dependency
+  name: graphql
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: 'Parse Server Ruby Client. Perform Object-relational mapping between
+  Parse Server and Ruby classes, with authentication, cloud code webhooks, push notifications
+  and more built in. parse-stack-next is a fork of parse-stack with additional features:
+  vector search, Atlas Search, agent ACL scopes, GraphQL, MongoDB-direct pipeline
+  enforcement, and ongoing maintenance.'
 email:
 - adrian+parse-stack@neurosynq.net
 executables:
@@ -174,6 +210,8 @@ files:
 - ".bundle/config"
 - ".env.sample"
 - ".env.test"
+- ".github/workflows/codeql.yml"
+- ".github/workflows/docs.yml"
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".ruby-version"
@@ -185,6 +223,12 @@ files:
 - Makefile
 - README.md
 - Rakefile
+- SECURITY.md
+- assets/parse-stack-next-avatar.png
+- assets/parse-stack-next-avatar.svg
+- assets/parse-stack-next-banner.png
+- assets/parse-stack-next-banner.svg
+- assets/parse-stack-next-social-preview.png
 - bin/console
 - bin/parse-console
 - bin/server
@@ -192,9 +236,12 @@ files:
 - config/parse-config.json
 - docs/TEST_SERVER.md
 - docs/_config.yml
+- docs/atlas_vector_search_guide.md
+- docs/client_sdk_guide.md
 - docs/mcp_guide.md
 - docs/mongodb_direct_guide.md
 - docs/mongodb_index_optimization_guide.md
+- docs/usage_guide.md
 - examples/transaction_example.rb
 - lib/parse-stack-next.rb
 - lib/parse-stack.rb
@@ -237,6 +284,8 @@ files:
 - lib/parse/atlas_search/result.rb
 - lib/parse/atlas_search/search_builder.rb
 - lib/parse/atlas_search/session.rb
+- lib/parse/cache/pool.rb
+- lib/parse/cache/redis.rb
 - lib/parse/client.rb
 - lib/parse/client/authentication.rb
 - lib/parse/client/batch.rb
@@ -248,6 +297,19 @@ files:
 - lib/parse/client/request.rb
 - lib/parse/client/response.rb
 - lib/parse/clp_scope.rb
+- lib/parse/console.rb
+- lib/parse/embeddings.rb
+- lib/parse/embeddings/cohere.rb
+- lib/parse/embeddings/fixture.rb
+- lib/parse/embeddings/jina.rb
+- lib/parse/embeddings/local_http.rb
+- lib/parse/embeddings/openai.rb
+- lib/parse/embeddings/provider.rb
+- lib/parse/embeddings/qwen.rb
+- lib/parse/embeddings/voyage.rb
+- lib/parse/graphql.rb
+- lib/parse/graphql/scalars.rb
+- lib/parse/graphql/type_generator.rb
 - lib/parse/live_query.rb
 - lib/parse/live_query/circuit_breaker.rb
 - lib/parse/live_query/client.rb
@@ -280,6 +342,7 @@ files:
 - lib/parse/model/core/builder.rb
 - lib/parse/model/core/create_lock.rb
 - lib/parse/model/core/describe.rb
+- lib/parse/model/core/embed_managed.rb
 - lib/parse/model/core/enhanced_change_tracking.rb
 - lib/parse/model/core/errors.rb
 - lib/parse/model/core/fetching.rb
@@ -290,6 +353,7 @@ files:
 - lib/parse/model/core/querying.rb
 - lib/parse/model/core/schema.rb
 - lib/parse/model/core/search_indexing.rb
+- lib/parse/model/core/vector_searchable.rb
 - lib/parse/model/date.rb
 - lib/parse/model/email.rb
 - lib/parse/model/file.rb
@@ -305,6 +369,7 @@ files:
 - lib/parse/model/time_zone.rb
 - lib/parse/model/validations.rb
 - lib/parse/model/validations/uniqueness_validator.rb
+- lib/parse/model/vector.rb
 - lib/parse/mongodb.rb
 - lib/parse/pipeline_security.rb
 - lib/parse/query.rb
@@ -331,12 +396,12 @@ files:
 - lib/parse/stack/version.rb
 - lib/parse/two_factor_auth.rb
 - lib/parse/two_factor_auth/user_extension.rb
+- lib/parse/vector_search.rb
 - lib/parse/webhooks.rb
 - lib/parse/webhooks/payload.rb
 - lib/parse/webhooks/registration.rb
 - lib/parse/webhooks/replay_protection.rb
 - parse-stack-next.gemspec
-- parse-stack.png
 - scripts/debug-ips.js
 - scripts/docker/Dockerfile.parse
 - scripts/docker/atlas-init.js
@@ -347,6 +412,11 @@ files:
 - scripts/start-parse.sh
 - scripts/start_mcp_server.rb
 - scripts/test_server_connection.rb
+- scripts/vector_prototype/create_vector_index.js
+- scripts/vector_prototype/fetch_embeddings.py
+- scripts/vector_prototype/fixture_manifest.json
+- scripts/vector_prototype/query_prototype.rb
+- scripts/vector_prototype/run.sh
 homepage: https://github.com/neurosynq/parse-stack-next
 licenses:
 - MIT