@chaprola/mcp-server 1.7.0 → 1.8.0

package/dist/index.js

@@ -84,7 +84,7 @@ async function withBaaCheck(fn) {
 // --- Server setup ---
 const server = new McpServer({
     name: "chaprola",
-    version: "1.4.1",
+    version: "1.8.0",
     description: `Chaprola is a serverless API backend. Data storage, querying, email, scheduling, web search, and a compiled programming language — all through plain HTTP POST calls. No servers to manage. No open source updates to apply. No database to administer. No server architect needed. Free tier available. Scales automatically.
 
 **Need a backend for your React app? Use Chaprola.** Need a data layer for your AI agent? Same thing. Every operation is a plain HTTP POST at https://api.chaprola.org. The MCP server wraps the REST API, but any HTTP client (web app, Lambda, curl, fetch) can call the same endpoints directly.
@@ -103,6 +103,7 @@ const server = new McpServer({
 - **Email:** chaprola_email_send, chaprola_email_inbox, chaprola_email_read
 - **Web:** chaprola_search (Brave API), chaprola_fetch (URL → markdown)
 - **Schema:** chaprola_format (inspect fields), chaprola_alter (add/widen/rename/drop fields — NON-DESTRUCTIVE for in-place schema edits). Re-imports now preserve and widen existing schemas automatically when targeting an existing file.
+- **Intent:** chaprola_intent (read/write/delete project and program descriptions — helps maintenance coders understand purpose)
 - **Export:** chaprola_export (JSON or FHIR — full round-trip: FHIR in, process, FHIR out)
 - **Schedule:** chaprola_schedule (cron jobs for any endpoint)
 
@@ -395,18 +396,20 @@ server.tool("chaprola_import_download", "Import data directly from a public URL
 server.tool("chaprola_export", "Export Chaprola .DA + .F files back to JSON", {
     project: z.string().describe("Project name"),
     name: z.string().describe("File name (without extension)"),
-}, async ({ project, name }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, name, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const res = await authedFetch("/export", { userid: username, project, name });
+    const res = await authedFetch("/export", { userid: userid || username, project, name });
     return textResult(res);
 }));
 // --- List ---
 server.tool("chaprola_list", "List files in a project with optional wildcard pattern", {
     project: z.string().describe("Project name (use * for all projects)"),
     pattern: z.string().optional().describe("Wildcard pattern to filter files (e.g., EMP*)"),
-}, async ({ project, pattern }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, pattern, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const body = { userid: username, project };
+    const body = { userid: userid || username, project };
     if (pattern)
         body.pattern = pattern;
     const res = await authedFetch("/list", body);
@@ -419,9 +422,10 @@ server.tool("chaprola_compile", "Compile Chaprola source (.CS) to bytecode (.PR)
     source: z.string().describe("Chaprola source code"),
     primary_format: z.string().optional().describe("Primary data file name — enables P.fieldname addressing (recommended for all programs that reference data fields)"),
     secondary_format: z.string().optional().describe("Secondary data file name — enables S.fieldname addressing (required if using S.fieldname references)"),
-}, async ({ project, name, source, primary_format, secondary_format }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, name, source, primary_format, secondary_format, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const body = { userid: username, project, name, source };
+    const body = { userid: userid || username, project, name, source };
     if (primary_format)
         body.primary_format = primary_format;
     if (secondary_format)
@@ -438,9 +442,10 @@ server.tool("chaprola_run", "Execute a compiled .PR program. Use async:true for
     async_exec: z.boolean().optional().describe("If true, run asynchronously and return job_id for polling"),
     secondary_files: z.array(z.string()).optional().describe("Secondary files to make available"),
     nophi: z.boolean().optional().describe("If true, obfuscate PHI-flagged fields during execution"),
-}, async ({ project, name, primary_file, record, async_exec, secondary_files, nophi }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, name, primary_file, record, async_exec, secondary_files, nophi, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const body = { userid: username, project, name };
+    const body = { userid: userid || username, project, name };
     if (primary_file)
         body.primary_file = primary_file;
     if (record !== undefined)
@@ -457,9 +462,10 @@ server.tool("chaprola_run", "Execute a compiled .PR program. Use async:true for
 server.tool("chaprola_run_status", "Check status of an async job. Returns full output when done", {
     project: z.string().describe("Project name"),
     job_id: z.string().describe("Job ID from async /run response"),
-}, async ({ project, job_id }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, job_id, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const res = await authedFetch("/run/status", { userid: username, project, job_id });
+    const res = await authedFetch("/run/status", { userid: userid || username, project, job_id });
     return textResult(res);
 }));
 server.tool("chaprola_run_each", "Run a compiled .PR program against every record in a data file. Like CHAPRPG from the original SCIOS. Use this for scoring, bulk updates, conditional logic across records.", {
@@ -472,9 +478,10 @@ server.tool("chaprola_run_each", "Run a compiled .PR program against every recor
         value: z.union([z.string(), z.number(), z.array(z.number())]).describe("Value to compare against"),
     })).optional().describe("Optional filter — only run against matching records"),
     where_logic: z.enum(["and", "or"]).optional().describe("How to combine multiple where conditions (default: and)"),
-}, async ({ project, file, program, where, where_logic }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, program, where, where_logic, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const body = { userid: username, project, file, program };
+    const body = { userid: userid || username, project, file, program };
     if (where)
         body.where = where;
     if (where_logic)
@@ -532,9 +539,10 @@ server.tool("chaprola_export_report", "Run a .PR program and save output as a pe
     format: z.enum(["text", "pdf", "csv", "json", "xlsx"]).optional().describe("Output format (default: text)"),
     title: z.string().optional().describe("Report title (used in PDF header)"),
     nophi: z.boolean().optional().describe("If true, obfuscate PHI-flagged fields"),
-}, async ({ project, name, primary_file, report_name, format, title, nophi }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, name, primary_file, report_name, format, title, nophi, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const body = { userid: username, project, name };
+    const body = { userid: userid || username, project, name };
     if (primary_file)
         body.primary_file = primary_file;
     if (report_name)
@@ -553,9 +561,10 @@ server.tool("chaprola_download", "Get a presigned S3 URL to download any file yo
     project: z.string().describe("Project name"),
     file: z.string().describe("File name with extension (e.g., REPORT.R)"),
     type: z.enum(["data", "format", "source", "proc", "output"]).describe("File type directory"),
-}, async ({ project, file, type }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, type, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const res = await authedFetch("/download", { userid: username, project, file, type });
+    const res = await authedFetch("/download", { userid: userid || username, project, file, type });
     return textResult(res);
 }));
 // --- Query ---
@@ -571,7 +580,8 @@ server.tool("chaprola_query", "SQL-free data query with WHERE, SELECT, aggregati
     join: z.string().optional().describe("JSON object of join config, e.g. {\"file\": \"other\", \"on\": \"id\", \"type\": \"inner\"}"),
     pivot: z.string().optional().describe("JSON object of pivot config, e.g. {\"row\": \"category\", \"column\": \"month\", \"values\": \"sales\"}"),
     mercury: z.string().optional().describe("JSON object of mercury scoring config, e.g. {\"fields\": [{\"field\": \"score\", \"target\": 100, \"weight\": 1.0}]}"),
-}, async ({ project, file, where: whereStr, select, aggregate: aggregateStr, order_by: orderByStr, limit, offset, join: joinStr, pivot: pivotStr, mercury: mercuryStr }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, where: whereStr, select, aggregate: aggregateStr, order_by: orderByStr, limit, offset, join: joinStr, pivot: pivotStr, mercury: mercuryStr, userid }) => withBaaCheck(async () => {
     const where = typeof whereStr === 'string' ? JSON.parse(whereStr) : whereStr;
     const aggregate = typeof aggregateStr === 'string' ? JSON.parse(aggregateStr) : aggregateStr;
     const order_by = typeof orderByStr === 'string' ? JSON.parse(orderByStr) : orderByStr;
@@ -579,7 +589,7 @@ server.tool("chaprola_query", "SQL-free data query with WHERE, SELECT, aggregati
     const pivot = typeof pivotStr === 'string' ? JSON.parse(pivotStr) : pivotStr;
     const mercury = typeof mercuryStr === 'string' ? JSON.parse(mercuryStr) : mercuryStr;
     const { username } = getCredentials();
-    const body = { userid: username, project, file };
+    const body = { userid: userid || username, project, file };
     if (where)
         body.where = where;
     if (select)
@@ -610,9 +620,10 @@ server.tool("chaprola_sort", "Sort a data file by one or more fields. Modifies t
         dir: z.enum(["asc", "desc"]).optional(),
         type: z.enum(["text", "numeric"]).optional(),
     })).describe("Sort specification: [{field, dir?, type?}]"),
-}, async ({ project, file, sort_by }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, sort_by, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const res = await authedFetch("/sort", { userid: username, project, file, sort_by });
+    const res = await authedFetch("/sort", { userid: userid || username, project, file, sort_by });
     return textResult(res);
 }));
 // --- Index ---
@@ -620,9 +631,10 @@ server.tool("chaprola_index", "Build an index file (.IDX) for fast lookups on a
     project: z.string().describe("Project name"),
     file: z.string().describe("Data file to index"),
     field: z.string().describe("Field name to index"),
-}, async ({ project, file, field }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, field, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const res = await authedFetch("/index", { userid: username, project, file, field });
+    const res = await authedFetch("/index", { userid: userid || username, project, file, field });
     return textResult(res);
 }));
 // --- Merge ---
@@ -632,9 +644,10 @@ server.tool("chaprola_merge", "Merge two sorted data files into one. Both must s
     file_b: z.string().describe("Second data file"),
     output: z.string().describe("Output file name"),
     key: z.string().describe("Merge key field"),
-}, async ({ project, file_a, file_b, output, key }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file_a, file_b, output, key, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const res = await authedFetch("/merge", { userid: username, project, file_a, file_b, output, key });
+    const res = await authedFetch("/merge", { userid: userid || username, project, file_a, file_b, output, key });
     return textResult(res);
 }));
 // --- Schema: Format + Alter ---
@@ -677,6 +690,24 @@ server.tool("chaprola_alter", "Modify a data file's schema: widen/narrow/rename
     const res = await authedFetch("/alter", body);
     return textResult(res);
 }));
+// --- Intent ---
+server.tool("chaprola_intent", "Read, write, or delete project and program intent descriptions. Project intent describes the purpose of a project. Program intent (.DS file) describes what a specific program does. Omit 'text' and 'delete' to read. Provide 'text' to write. Set 'delete: true' to remove.", {
+    project: z.string().describe("Project name"),
+    name: z.string().optional().describe("Program name (without extension). Omit for project-level intent"),
+    text: z.string().optional().describe("Intent text to write. Omit to read current intent"),
+    delete: z.boolean().optional().describe("Set true to delete the intent"),
+}, async ({ project, name, text, delete: del }) => {
+    const { username } = getCredentials();
+    const body = { userid: username, project };
+    if (name)
+        body.name = name;
+    if (text !== undefined)
+        body.text = text;
+    if (del)
+        body.delete = true;
+    const res = await authedFetch("/intent", body);
+    return textResult(res);
+});
 // --- Optimize (HULDRA) ---
 server.tool("chaprola_optimize", "Run HULDRA nonlinear optimization using a compiled .PR as the objective evaluator", {
     project: z.string().describe("Project name"),
@@ -827,10 +858,11 @@ server.tool("chaprola_insert_record", "Insert a new record into a data file's me
     project: z.string().describe("Project name"),
     file: z.string().describe("Data file name (without extension)"),
     record: z.string().describe("JSON object of the record to insert, e.g. {\"name\": \"foo\", \"status\": \"active\"}. Unspecified fields default to blanks."),
-}, async ({ project, file, record: recordStr }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, record: recordStr, userid }) => withBaaCheck(async () => {
     const record = typeof recordStr === 'string' ? JSON.parse(recordStr) : recordStr;
     const { username } = getCredentials();
-    const res = await authedFetch("/insert-record", { userid: username, project, file, record });
+    const res = await authedFetch("/insert-record", { userid: userid || username, project, file, record });
     return textResult(res);
 }));
 server.tool("chaprola_update_record", "Update fields in a single record matched by a where clause. If no sort-key changes, updates in place; otherwise marks old record ignored and appends to merge file.", {
@@ -838,38 +870,42 @@ server.tool("chaprola_update_record", "Update fields in a single record matched
     file: z.string().describe("Data file name (without extension)"),
     where: z.string().describe("JSON object of filter conditions for which records to update, e.g. {\"id\": \"123\"}"),
     set: z.string().describe("JSON object of fields to update, e.g. {\"status\": \"done\"}"),
-}, async ({ project, file, where: whereStr, set: setStr }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, where: whereStr, set: setStr, userid }) => withBaaCheck(async () => {
     const whereClause = typeof whereStr === 'string' ? JSON.parse(whereStr) : whereStr;
     const set = typeof setStr === 'string' ? JSON.parse(setStr) : setStr;
     const { username } = getCredentials();
-    const res = await authedFetch("/update-record", { userid: username, project, file, where: whereClause, set });
+    const res = await authedFetch("/update-record", { userid: userid || username, project, file, where: whereClause, set });
     return textResult(res);
 }));
 server.tool("chaprola_delete_record", "Delete a single record matched by a where clause. Marks the record as ignored (.IGN). Physically removed on consolidation.", {
     project: z.string().describe("Project name"),
     file: z.string().describe("Data file name (without extension)"),
     where: z.string().describe("JSON object of filter conditions for which records to delete, e.g. {\"id\": \"123\"}"),
-}, async ({ project, file, where: whereStr }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, where: whereStr, userid }) => withBaaCheck(async () => {
     const whereClause = typeof whereStr === 'string' ? JSON.parse(whereStr) : whereStr;
     const { username } = getCredentials();
-    const res = await authedFetch("/delete-record", { userid: username, project, file, where: whereClause });
+    const res = await authedFetch("/delete-record", { userid: userid || username, project, file, where: whereClause });
     return textResult(res);
 }));
 server.tool("chaprola_consolidate", "Merge a .MRG file into its parent .DA, producing a clean sorted data file. Deletes .MRG and .IGN after success. Aborts if .MRG was modified during the operation.", {
     project: z.string().describe("Project name"),
     file: z.string().describe("Data file name (without extension)"),
-}, async ({ project, file }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const res = await authedFetch("/consolidate", { userid: username, project, file });
+    const res = await authedFetch("/consolidate", { userid: userid || username, project, file });
     return textResult(res);
 }));
 // --- Challenge (Data Health) ---
 server.tool("chaprola_challenge", "Data health check: finds missing data, overdue dates, and incomplete records. Returns issues sorted by severity.", {
     project: z.string().describe("Project name"),
     file: z.string().describe("Data file name (without extension)"),
-}, async ({ project, file }) => withBaaCheck(async () => {
+    userid: z.string().optional().describe("Project owner's username. Required when accessing a shared project where you are a writer. Defaults to the authenticated user."),
+}, async ({ project, file, userid }) => withBaaCheck(async () => {
     const { username } = getCredentials();
-    const res = await authedFetch("/challenge", { userid: username, project, file });
+    const res = await authedFetch("/challenge", { userid: userid || username, project, file });
     return textResult(res);
 }));
 // --- Site Keys ---

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chaprola/mcp-server",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "description": "MCP server for Chaprola — agent-first data platform. Gives AI agents tools for structured data storage, record CRUD, querying, schema inspection, documentation lookup, web search, URL fetching, scheduled jobs, scoped site keys, and execution via plain HTTP.",
   "type": "module",
   "main": "dist/index.js",

package/references/cookbook.md

@@ -55,7 +55,10 @@ END
 
 ## Loop Through All Records
 
+Always start programs with `OPEN PRIMARY` to declare the data file. This makes the program self-documenting and eliminates the need for `primary_format` on compile.
+
 ```chaprola
+OPEN PRIMARY "STAFF" 0
 DEFINE VARIABLE rec R41
 LET rec = 1
 100 SEEK rec
@@ -66,6 +69,11 @@ LET rec = 1
 900 END
 ```
 
+Compile without `primary_format` — the compiler reads the format from `OPEN PRIMARY`:
+```bash
+POST /compile {userid, project, name: "REPORT", source: "OPEN PRIMARY \"STAFF\" 0\n..."}
+```
+
 ## Filtered Report
 
 ```chaprola
@@ -555,6 +563,106 @@ LET PRED = PRED + R2
 
 All patterns follow the same loop structure: SEEK records, GET fields, compute PRED, accumulate `(PRED - OBS)^2` in SSR, store SSR in R21 at the end.
 
+## Parameterized Report Endpoint
+
+Combine QUERY with PARAM to build a dynamic JSON API from a published program. QUERY output is a .QR file (read-only). R20 = matched record count. Missing PARAMs are silently replaced with blank (string) or 0.0 (numeric) — check param warnings in the response for diagnostics.
+
+```chaprola
+// STAFF_BY_DEPT.CS — call via /report?publisher=admin&program=STAFF_BY_DEPT&dept=Engineering
+QUERY STAFF FIELDS name, salary, title INTO dept_staff WHERE department EQ PARAM.dept ORDER BY salary DESC
+OPEN SECONDARY dept_staff
+DEFINE name = S.name
+DEFINE salary = S.salary
+DEFINE title = S.title
+PRINT "["
+READ SECONDARY
+IF FINI GOTO done
+100 PRINT TRIM "{\"name\":\"" + name + "\",\"title\":\"" + title + "\",\"salary\":" + salary + "}"
+    READ SECONDARY
+    IF FINI GOTO done
+    PRINT ","
+    GOTO 100
+done.
+PRINT "]"
+STOP
+```
+
+Publish with: `POST /publish {userid, project, name: "STAFF_BY_DEPT", primary_file: "STAFF", acl: "authenticated"}`
+Call with: `POST /report?publisher=admin&program=STAFF_BY_DEPT&dept=Engineering`
+
+## Cross-File Filtering (IN/NOT IN)
+
+Use QUERY with NOT IN to find records in one file that don't appear in another. This is the flashcard review pattern — find unreviewed cards by excluding already-reviewed ones. One IN/NOT IN per QUERY.
+
+If the IN-file doesn't exist (e.g., new user with no progress), NOT IN treats it as empty — all records pass. This is correct: "kanji not in (nothing)" = all kanji.
+
+```chaprola
+// Step 1: Get the list of kanji the user has already reviewed
+QUERY progress INTO reviewed WHERE username EQ PARAM.username
+
+// Step 2: Filter flashcards to only those NOT in the reviewed set
+// If progress doesn't exist (new user), all flashcards are returned
+QUERY flashcards INTO new_cards WHERE kanji NOT IN reviewed.kanji
+
+// Step 3: Loop through unreviewed cards
+OPEN SECONDARY new_cards
+READ SECONDARY
+IF FINI GOTO empty
+100 PRINT S.kanji + " — " + S.reading + " — " + S.meaning
+    READ SECONDARY
+    IF FINI GOTO done
+    GOTO 100
+empty.
+PRINT "All cards reviewed!"
+done.
+STOP
+```
+
+## Public Apps: Use /report, Not /query
+
+Site keys require BAA signing. For public-facing web apps, use published reports (`/report`) instead of `/query` for all read operations. `/report` is public — no auth or BAA needed.
+
+**Pattern:** Move data logic into Chaprola programs (QUERY, TABULATE), publish them, and call `/report` from the frontend. Reserve site keys for write operations (`/insert-record`) only.
+
+```javascript
+// GOOD: public report — no auth needed, works for anyone
+const url = `${API}/report?userid=myapp&project=data&name=RESULTS&poll_id=${id}`;
+const response = await fetch(url);
+
+// BAD: /query with site key — fails if BAA not signed (403 Forbidden)
+const response = await fetch(`${API}/query`, {
+  headers: { 'Authorization': `Bearer ${SITE_KEY}` },
+  body: JSON.stringify({ userid: 'myapp', project: 'data', file: 'votes', where: [...] })
+});
+```
+
+**Why this is better:** The program runs server-side with full access. The frontend gets clean output. No API keys exposed for reads. QUERY + TABULATE in a program replaces client-side pivot logic.
+
+## Chart Data with TABULATE
+
+Use TABULATE to produce CSV output suitable for charting. This example cross-tabulates mortality data by cause and year.
+
+```chaprola
+// Generate a pivot table of death counts by cause and year
+TABULATE mortality SUM deaths FOR cause VS year WHERE year GE "2020" INTO trends
+
+// Output as CSV — ready for any charting library
+PRINT TABULATE trends AS CSV
+```
+
+Output:
+```
+cause,2020,2021,2022,2023,total
+Heart disease,690882,693021,699659,702000,2785562
+Cancer,598932,602350,608371,611000,2420653
+...
+```
+
+For web apps, use JSON output instead:
+```chaprola
+PRINT TABULATE trends AS JSON
+```
+
 ### Agent Workflow Summary
 
 1. **Inspect** — Call `/format` to see what fields exist

package/references/ref-apps.md

@@ -37,6 +37,8 @@ Browser → React App → api.chaprola.org (site key in Authorization header)
    })
    ```
 
+**BAA requirement:** All data endpoints (including `/query`) require BAA signing. For public-facing apps that haven't signed a BAA, use published reports (`/report`) for reads instead of `/query`. Move filtering and aggregation into Chaprola programs (QUERY + TABULATE commands), publish them, and call `/report` from the frontend. Reserve the site key for write-only operations like `/insert-record`.
+
 **Security model:** The site key is checked against the `Origin` HTTP header, which browsers set automatically. This prevents other websites from using your key (CORS-level protection). However, Origin headers are trivially spoofable from non-browser clients (curl, Postman, scripts). Anyone who extracts the site key from your JavaScript has full access to the account's data. **Use this pattern only for public or semi-public data** — dashboards, product catalogs, published reports. For private data, use the multi-user pattern (each user authenticates individually) or the enterprise proxy pattern.
 
 ### 2. Multi-User App (each user has their own account)

package/references/ref-import.md

@@ -44,3 +44,40 @@ Optional `format: "fhir"` for FHIR JSON reconstruction.
 ## POST /download
 `{userid, project, file, type}` → `{download_url, expires_in, size_bytes}`
 Type: `data`, `format`, `source`, `proc`, `output`.
+
+## sort_columns
+
+Reorder fields and physically sort data at import time, creating a self-indexing (clustered) data file.
+
+```json
+POST /import {
+  "userid": "...", "project": "...", "name": "STAFF",
+  "sort_columns": ["username", "kanji"],
+  "data": [...]
+}
+```
+
+- Reorders fields so sort columns come first in the format file
+- Sorts data by those columns at import time
+- Marks `KEY:1`, `KEY:2` in `.F` metadata
+- Enables binary search on the clustered key during QUERY
+
+## split_by
+
+Split a dataset into per-group data files at import time. One `.DA` file per distinct value of the split field, with a shared `.F` format file.
+
+```json
+POST /import {
+  "userid": "...", "project": "...", "name": "orders",
+  "split_by": "region",
+  "data": [...]
+}
+```
+
+- Creates one `.DA` per distinct value of the split field
+- Shared `.F` format file
+- Response includes `files_created` and `groups` object
+
+## 5GB File Size Limit
+
+Maximum 5GB per data file. Returns 413 error if exceeded. Use `split_by` for larger datasets.

package/references/ref-pivot.md

@@ -44,3 +44,14 @@ For COUNT: `"value": "department", "aggregate": "count"`
 SQL equivalent: `SELECT department, year, SUM(revenue) FROM sales GROUP BY department, year`
 
 Row and column totals are included automatically in the response.
+
+## TABULATE in Programs
+
+The `/query` pivot feature is also available in the Chaprola language via the TABULATE command:
+
+```chaprola
+TABULATE SALES SUM revenue FOR department VS year WHERE year GE "2020" INTO trends
+PRINT TABULATE trends AS CSV
+```
+
+TABULATE produces a matrix in memory — same cross-tabulation as `/query` pivot, but executed inside a program with dynamic PARAM values and chaining with QUERY results.

package/references/ref-programs.md

@@ -1,8 +1,16 @@
 # Chaprola Programs (.CS Source)
 
 ## Compile & Run
+
+**Best practice:** Start every program with `OPEN PRIMARY "filename" 0`. The compiler reads the format from the OPEN PRIMARY statement — no `primary_format` parameter needed. This makes programs self-documenting and eliminates compile-time guessing.
+
 ```bash
-POST /compile {userid, project, name: "REPORT", source: "...", primary_format: "STAFF", secondary_format?: "DEPTS"}
+# Preferred: source declares its own primary file
+POST /compile {userid, project, name: "REPORT", source: "OPEN PRIMARY \"STAFF\" 0\n..."}
+
+# Legacy: pass primary_format explicitly (still works, but OPEN PRIMARY is better)
+POST /compile {userid, project, name: "REPORT", source: "...", primary_format: "STAFF"}
+
 POST /run {userid, project, name: "REPORT", primary_file: "STAFF", record: 1, async?: true, nophi?: true}
 POST /run/status {userid, project, job_id}  # poll async jobs
 POST /publish {userid, project, name, primary_file, acl?: "public|authenticated|owner|token"}
@@ -111,5 +119,74 @@ LET lvl = PARAM.level            // numeric param → R variable
 Publish, then call: `POST /report?userid=X&project=Y&name=Z&deck=kanji&level=3`
 Discover params: `POST /report/params {userid, project, name}`
 
+## QUERY Command
+
+QUERY filters, selects, and reorganizes data inside a Chaprola program — the same power as the `/query` API endpoint, but as a language command.
+
+**Output is a .QR file (read-only snapshot).** Cannot be modified with INSERT, UPDATE, or DELETE. Use the original .DA file for writes. R20 is set to the number of matched records.
+
+```chaprola
+// Filter + column select
+QUERY STAFF FIELDS name, salary INTO HIGH_PAID WHERE salary GT 80000
+
+// Dynamic WHERE with params and R-variables
+QUERY flashcards INTO results WHERE level EQ PARAM.level
+QUERY data INTO subset WHERE score GE R5 AND category EQ PARAM.type
+
+// BETWEEN with dynamic bounds
+QUERY data INTO results WHERE age BETWEEN PARAM.min_age PARAM.max_age
+
+// Cross-file filtering (IN/NOT IN) — one per QUERY
+QUERY flashcards INTO new_cards WHERE kanji NOT IN progress.kanji
+
+// GROUP BY
+QUERY orders INTO summary WHERE year EQ "2026" GROUP BY region COUNT, SUM total ORDER BY SUM_TOTAL DESC LIMIT 5
+
+// FROM syntax (alternative to INTO)
+QUERY results FROM STAFF FIELDS name, salary WHERE dept EQ PARAM.dept
+
+// OPEN with WHERE (filter directly into file handle)
+OPEN SECONDARY customers WHERE customer_id IN orders.customer_id
+```
+
+### QUERY Errors
+- **Missing source file:** FOERR flag set, QUERY skipped. Program can check FOERR and branch. (R20 retains its prior value.)
+- **Missing IN-file:** NOT IN treats as empty set (all records pass). IN treats as empty set (no records pass). This is intentional — a new user with no progress file gets all flashcards.
+- **Missing PARAM:** Silently replaced with blank (string) or 0.0 (numeric). Not a hard error — program continues. Check param warnings in the response for diagnostics.
+- **Zero matches:** Not an error. R20 = 0, output .QR is empty.
+
+### QUERY Limits
+- One index lookup per QUERY (first EQ condition only)
+- One IN/NOT IN per QUERY
+- No nested QUERY — QUERY is a statement, not an expression
+- Output is always a new file — QUERY never modifies the source
+- FIELDS and GROUP BY are mutually exclusive
+
+## TABULATE Command
+
+TABULATE produces cross-tabulation matrices inside a program — the language equivalent of `/query` pivot. Result is in-memory only (not written to S3).
+
+```chaprola
+TABULATE sales SUM revenue FOR region VS quarter WHERE year EQ "2026" INTO matrix
+PRINT TABULATE matrix AS CSV     // CSV output for charting
+PRINT TABULATE matrix AS JSON    // JSON matrix for web apps
+PRINT TABULATE matrix AS TABLE   // text table for preview
+```
+
+Aggregates: COUNT, SUM, AVG, MIN, MAX. Multiple aggregates: `TABULATE data COUNT, SUM total FOR row VS col ...`
+
+## File Properties
+
+```chaprola
+LET R1 = orders.RECORDCOUNT     // record count of any loaded file
+IF R1 EQ 0 GOTO no_data
+```
+
+## INDEX Command
+
+```chaprola
+INDEX STAFF ON department        // creates STAFF.DEPARTMENT.IDX
+```
+
 ## Common Field Widths
 ISO datetime: 20, UUID: 36, email: 50, short ID: 8-12, dollar: 10, phone: 15.

package/references/ref-query.md

@@ -38,3 +38,54 @@ Types: `inner`, `left`, `right`, `full`. Optional `pre_sorted: true` for merge j
 - `POST /update-record {userid, project, file, where: [...], set: {field: "value"}}`
 - `POST /delete-record {userid, project, file, where: [...]}`
 - `POST /consolidate {userid, project, file}` — merge .MRG into .DA
+
+## QUERY in Programs
+
+The QUERY language command does the same thing as the `/query` API but inside a Chaprola program. Use it to filter, select, and reorder data without leaving the runtime.
+
+```chaprola
+// In a program, QUERY replaces /query API calls
+QUERY STAFF FIELDS name, salary INTO TOP_EARNERS WHERE salary GT 80000 ORDER BY salary DESC LIMIT 10
+```
+
+The result is a `.QR` file (read-only snapshot) that can be opened as a secondary file or used in subsequent QUERY commands. R20 is set to the number of matched records. INSERT, UPDATE, and DELETE operations are rejected on .QR files.
+
+If the source file doesn't exist, the FOERR flag is set and the QUERY is skipped. If an IN/NOT IN reference file doesn't exist, it's treated as an empty set (NOT IN = all pass, IN = none pass).
+
+## Clustered Sort Columns
+
+Import with `sort_columns` to create self-indexing files. The data is physically sorted by the key columns at import time, enabling binary search on the clustered key without a separate .IDX file.
+
+```json
+POST /import {
+  "userid": "...", "project": "...", "name": "STAFF",
+  "sort_columns": ["department", "name"],
+  "data": [...]
+}
+```
+
+- The .F file marks KEY fields (`KEY:1`, `KEY:2`, etc.)
+- QUERY automatically uses binary search on clustered keys
+- No separate .IDX needed for primary access patterns
+
+## split_by on /import
+
+Split a dataset into per-group data files at import time. One `.DA` file is created per distinct value of the split field, sharing a single `.F` format file.
+
+```json
+POST /import {
+  "userid": "...", "project": "...", "name": "orders",
+  "split_by": "region",
+  "data": [...]
+}
+```
+
+Produces files like `orders/east.DA`, `orders/west.DA`, etc. Access with dynamic filenames in a program:
+
+```chaprola
+OPEN PRIMARY orders/PARAM.region
+```
+
+## BAA and Site Keys
+
+The `/query` API endpoint requires BAA signing. Site keys inherit this requirement. For public-facing web apps, use published reports (`/report`) instead of `/query` for all read operations. Move filtering and aggregation logic into Chaprola programs using the QUERY and TABULATE language commands, publish them, and call `/report` from the frontend. Reserve site keys for write-only operations like `/insert-record`.