@san-francisco/sf-docs-embeddings 0.0.3

package/LICENSE

@@ -0,0 +1,20 @@
+The San Francisco License (SF License)
+
+Copyright (c) San Francisco License Contributors 2026
+
+Permission is granted, free of charge, to any person or organization
+obtaining a copy of this software and associated documentation files
+(the "Software"), to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit others
+to whom the Software is furnished to do the same.
+
+The only requirement is that this license notice and copyright notice
+shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT, OR OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/Makefile

@@ -0,0 +1,6 @@
+EXTENSION = sf-docs-embeddings
+DATA = sql/sf-docs-embeddings--0.0.1.sql
+
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)

package/README.md

@@ -0,0 +1,59 @@
+# @san-francisco/sf-docs-embeddings
+
+San Francisco documentation embeddings - example RAG data package.
+
+## Overview
+
+This package demonstrates how to ship pre-computed embeddings as PGPM migrations. It creates a sample collection with San Francisco city documentation and example embeddings.
+
+## What This Package Does
+
+1. Creates an embedding model configuration for `text-embedding-3-small`
+2. Creates the `sf-docs` collection with semantic chunking config
+3. Seeds example documents and chunks
+4. (In production) Would include actual vector embeddings
+
+## Usage
+
+```bash
+# Install the RAG schema first
+pgpm deploy @sf-bot/rag-core
+
+# Then install this data package
+pgpm deploy @san-francisco/sf-docs-embeddings
+```
+
+## Data Structure
+
+After installation, you'll have:
+
+- **Collection**: `sf-docs` - San Francisco city documentation
+- **Model**: `text-embedding-3-small` (OpenAI, 1536 dimensions)
+- **Documents**: Example SF city services content
+- **Chunks**: Semantically chunked document segments
+
+## Creating Your Own Data Package
+
+To create a similar data package for your own embeddings:
+
+1. Generate embeddings using your preferred model
+2. Export using `rag.export_collection_json()`
+3. Convert the JSON to SQL INSERT statements
+4. Package as a PGPM module
+
+Example workflow:
+```sql
+-- Export your collection
+SELECT rag.export_collection_json('your-collection-id');
+
+-- Or export as CSV for processing
+SELECT * FROM rag.export_embeddings_csv('your-collection-id');
+```
+
+## Dependencies
+
+- `@sf-bot/rag-core`
+
+## License
+
+[SF License](https://github.com/city-of-san-francisco/license)

package/deploy/data/seed_collection.sql

@@ -0,0 +1,93 @@
+-- Deploy data/seed_collection to pg
+-- made with <3 @ constructive.io
+
+-- requires: @sf-bot/rag-core
+
+-- This is an example data package showing how to ship embeddings as migrations.
+-- In production, this would contain actual embeddings generated from SF documentation.
+
+BEGIN;
+
+-- Insert embedding model (if not exists)
+INSERT INTO rag.embedding_model (name, provider, dimensions, metadata)
+VALUES (
+  'text-embedding-3-small',
+  'openai',
+  1536,
+  '{"description": "OpenAI text-embedding-3-small model"}'::jsonb
+)
+ON CONFLICT (name) DO NOTHING;
+
+-- Create the SF Docs collection
+INSERT INTO rag.collection (name, description, chunk_config, metadata)
+VALUES (
+  'sf-docs',
+  'San Francisco city documentation and public information',
+  '{"strategy": "semantic", "max_tokens": 512, "overlap_tokens": 50}'::jsonb,
+  '{"source": "sf.gov", "version": "2026.01"}'::jsonb
+)
+ON CONFLICT (name) DO NOTHING;
+
+-- Link collection to model
+INSERT INTO rag.collection_model (collection_id, model_id, is_default)
+SELECT c.id, m.id, true
+FROM rag.collection c, rag.embedding_model m
+WHERE c.name = 'sf-docs' AND m.name = 'text-embedding-3-small'
+ON CONFLICT (collection_id, model_id) DO NOTHING;
+
+-- Example document: SF City Services Overview
+-- In production, this would be populated with actual content and embeddings
+INSERT INTO rag.document (collection_id, source_uri, source_type, content, content_hash, metadata, status)
+SELECT 
+  c.id,
+  'https://sf.gov/services',
+  'url',
+  'San Francisco provides a wide range of city services to residents and visitors. These include public transportation through MUNI and BART, parks and recreation facilities, public libraries, and emergency services. The city also offers various permits and licenses for businesses and residents.',
+  'sha256:example_sf_services_001',
+  '{"title": "SF City Services Overview", "category": "services"}'::jsonb,
+  'embedded'
+FROM rag.collection c
+WHERE c.name = 'sf-docs'
+ON CONFLICT (collection_id, content_hash) DO NOTHING;
+
+-- Example chunks for the document
+-- Note: In production, these would be generated by a chunking algorithm
+WITH doc AS (
+  SELECT d.id FROM rag.document d
+  JOIN rag.collection c ON c.id = d.collection_id
+  WHERE c.name = 'sf-docs' AND d.content_hash = 'sha256:example_sf_services_001'
+)
+INSERT INTO rag.chunk (document_id, content, chunk_index, start_offset, end_offset, token_count, metadata)
+SELECT 
+  doc.id,
+  'San Francisco provides a wide range of city services to residents and visitors. These include public transportation through MUNI and BART, parks and recreation facilities, public libraries, and emergency services.',
+  0,
+  0,
+  220,
+  42,
+  '{"section": "introduction"}'::jsonb
+FROM doc
+ON CONFLICT (document_id, chunk_index) DO NOTHING;
+
+WITH doc AS (
+  SELECT d.id FROM rag.document d
+  JOIN rag.collection c ON c.id = d.collection_id
+  WHERE c.name = 'sf-docs' AND d.content_hash = 'sha256:example_sf_services_001'
+)
+INSERT INTO rag.chunk (document_id, content, chunk_index, start_offset, end_offset, token_count, metadata)
+SELECT 
+  doc.id,
+  'The city also offers various permits and licenses for businesses and residents.',
+  1,
+  221,
+  300,
+  14,
+  '{"section": "permits"}'::jsonb
+FROM doc
+ON CONFLICT (document_id, chunk_index) DO NOTHING;
+
+-- Example embeddings (placeholder vectors - in production these would be real embeddings)
+-- Using 3-dimensional vectors for this example; real embeddings would be 1536-dimensional
+-- Note: The actual embedding insertion would use the full 1536-dim vectors
+
+COMMIT;

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@san-francisco/sf-docs-embeddings",
+  "version": "0.0.3",
+  "author": "Dan Lynch <pyramation@gmail.com>",
+  "description": "San Francisco documentation embeddings - example RAG data package",
+  "homepage": "https://github.com/constructive-io/sf-bot",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/constructive-io/sf-bot"
+  },
+  "bugs": {
+    "url": "https://github.com/constructive-io/sf-bot/issues"
+  },
+  "files": [
+    "deploy",
+    "verify",
+    "revert",
+    "pgpm.plan",
+    "*.control",
+    "Makefile"
+  ],
+  "scripts": {
+    "build": "true",
+    "clean": "true",
+    "lint": "eslint . --fix",
+    "test": "jest",
+    "test:watch": "jest --watchAll"
+  },
+  "keywords": [
+    "san-francisco",
+    "embeddings",
+    "rag",
+    "data"
+  ],
+  "devDependencies": {
+    "makage": "0.1.9",
+    "pgsql-test": "^2.18.15"
+  },
+  "gitHead": "4f7e6a0f13025df722f24602bfb1d173d04cf95c"
+}

package/pgpm.plan

@@ -0,0 +1,5 @@
+%syntax-version=1.0.0
+%project=sf-docs-embeddings
+%uri=sf-docs-embeddings
+
+data/seed_collection 2026-01-25T01:10:00Z constructive <constructive@sf-bot> # seed SF docs collection with example data

package/revert/data/seed_collection.sql

@@ -0,0 +1,8 @@
+-- Revert data/seed_collection from pg
+
+BEGIN;
+
+-- Remove the SF Docs collection and all related data (cascades to documents, chunks, embeddings)
+DELETE FROM rag.collection WHERE name = 'sf-docs';
+
+COMMIT;

package/sf-docs-embeddings.control

@@ -0,0 +1,7 @@
+# sf-docs-embeddings extension
+comment = 'San Francisco documentation embeddings - example RAG data package'
+default_version = '0.0.1'
+module_pathname = '$libdir/sf-docs-embeddings'
+requires = 'rag-core'
+relocatable = false
+superuser = false

package/verify/data/seed_collection.sql

@@ -0,0 +1,7 @@
+-- Verify data/seed_collection on pg
+
+BEGIN;
+
+SELECT 1/count(*) FROM rag.collection WHERE name = 'sf-docs';
+
+ROLLBACK;