@xano/developer-mcp 1.0.8 → 1.0.9

package/xanoscript_docs/streaming.md

@@ -0,0 +1,318 @@
+---
+applyTo: "functions/**/*.xs, apis/**/*.xs"
+---
+
+# Streaming Operations
+
+Stream data from files, requests, and to API responses.
+
+## Quick Reference
+
+| Operation | Purpose | Input |
+|-----------|---------|-------|
+| `stream.from_csv` | Parse CSV stream | File or string |
+| `stream.from_jsonl` | Parse JSONL stream | File or string |
+| `stream.from_request` | Stream HTTP request body | Request |
+| `api.stream` | Stream response to client | Data |
+
+---
+
+## stream.from_csv
+
+Parse CSV data as a stream, processing rows one at a time without loading entire file into memory.
+
+```xs
+stream.from_csv {
+  value = $input.csv_file
+  headers = true
+  delimiter = ","
+  enclosure = "\""
+} as $row {
+  // Process each row
+  db.add "import_record" {
+    data = {
+      name: $row.name,
+      email: $row.email,
+      created_at: now
+    }
+  }
+}
+```
+
+### Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `value` | file/text | required | CSV file or string |
+| `headers` | bool | `true` | First row contains headers |
+| `delimiter` | text | `,` | Field delimiter |
+| `enclosure` | text | `"` | Field enclosure character |
+| `escape` | text | `\` | Escape character |
+
+### With Custom Headers
+
+```xs
+stream.from_csv {
+  value = $input.file
+  headers = false
+} as $row {
+  // Access by index: $row.0, $row.1, etc.
+  var $name { value = $row.0 }
+  var $email { value = $row.1 }
+}
+```
+
+---
+
+## stream.from_jsonl
+
+Parse JSON Lines (newline-delimited JSON) as a stream.
+
+```xs
+stream.from_jsonl {
+  value = $input.jsonl_file
+} as $record {
+  // Each $record is a parsed JSON object
+  db.add "event" {
+    data = {
+      event_type: $record.type,
+      payload: $record.data,
+      timestamp: $record.ts
+    }
+  }
+}
+```
+
+### Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `value` | file/text | required | JSONL file or string |
+
+### Example JSONL Format
+
+```
+{"type":"click","data":{"page":"/home"},"ts":1699900000}
+{"type":"view","data":{"page":"/products"},"ts":1699900001}
+{"type":"click","data":{"page":"/checkout"},"ts":1699900002}
+```
+
+---
+
+## stream.from_request
+
+Stream the incoming HTTP request body for large uploads.
+
+```xs
+stream.from_request {
+  format = "jsonl"
+} as $record {
+  // Process streamed data
+  db.add "log" { data = $record }
+}
+```
+
+### Parameters
+
+| Parameter | Type | Options | Description |
+|-----------|------|---------|-------------|
+| `format` | text | `jsonl`, `csv`, `raw` | Body format |
+| `headers` | bool | `true` (csv) | CSV has headers |
+| `delimiter` | text | `,` (csv) | CSV delimiter |
+
+### Raw Chunks
+
+```xs
+stream.from_request {
+  format = "raw"
+  chunk_size = 8192
+} as $chunk {
+  // Process raw bytes
+  storage.append_file {
+    pathname = "uploads/large_file.bin"
+    data = $chunk
+  }
+}
+```
+
+---
+
+## api.stream
+
+Stream response data to the client. Useful for large datasets, real-time feeds, or server-sent events.
+
+### Streaming JSONL Response
+
+```xs
+query "export_logs" {
+  input {
+    timestamp start_date
+    timestamp end_date
+  }
+  stack {
+    db.query "log" {
+      where = $db.log.created_at >= $input.start_date && $db.log.created_at <= $input.end_date
+      return = { type: "stream" }
+    } as $logs
+
+    api.stream {
+      format = "jsonl"
+      value = $logs
+    }
+  }
+}
+```
+
+### Streaming CSV Response
+
+```xs
+query "export_users_csv" {
+  stack {
+    db.query "user" {
+      return = { type: "stream" }
+    } as $users
+
+    api.stream {
+      format = "csv"
+      value = $users
+      headers = ["id", "name", "email", "created_at"]
+      filename = "users_export.csv"
+    }
+  }
+}
+```
+
+### Server-Sent Events (SSE)
+
+```xs
+query "live_updates" {
+  stack {
+    api.stream {
+      format = "sse"
+    } as $stream
+
+    // Send events to client
+    $stream.send { event = "connected", data = { status: "ok" } }
+
+    foreach ($updates) {
+      each as $update {
+        $stream.send { event = "update", data = $update }
+      }
+    }
+
+    $stream.close
+  }
+}
+```
+
+### Parameters
+
+| Parameter | Type | Options | Description |
+|-----------|------|---------|-------------|
+| `format` | text | `jsonl`, `csv`, `sse`, `raw` | Stream format |
+| `value` | stream | optional | Data source for jsonl/csv |
+| `headers` | text[] | optional | CSV column headers |
+| `filename` | text | optional | Suggested download filename |
+
+---
+
+## Streaming Patterns
+
+### Large File Import
+
+```xs
+function "import_large_csv" {
+  input {
+    file csv_file
+  }
+  stack {
+    var $processed { value = 0 }
+    var $errors { value = [] }
+
+    stream.from_csv {
+      value = $input.csv_file
+      headers = true
+    } as $row {
+      try_catch {
+        try {
+          db.add "record" {
+            data = {
+              name: $row.name|trim,
+              email: $row.email|trim|lower,
+              created_at: now
+            }
+          }
+          math.add $processed { value = 1 }
+        }
+        catch {
+          var.update $errors {
+            value = $errors|push:{ row: $processed, error: $error.message }
+          }
+        }
+      }
+    }
+  }
+  response = { processed: $processed, errors: $errors }
+}
+```
+
+### ETL Pipeline
+
+```xs
+function "etl_events" {
+  input {
+    file events_file
+  }
+  stack {
+    stream.from_jsonl {
+      value = $input.events_file
+    } as $event {
+      // Transform
+      var $transformed {
+        value = {
+          event_type: $event.type|to_lower,
+          user_id: $event.user_id|to_int,
+          metadata: $event.data|json_encode,
+          occurred_at: $event.timestamp|to_timestamp
+        }
+      }
+
+      // Load
+      db.add "processed_event" {
+        data = $transformed
+      }
+    }
+  }
+  response = { status: "complete" }
+}
+```
+
+### Chunked Export
+
+```xs
+query "stream_large_dataset" {
+  stack {
+    // Stream query results directly to response
+    db.query "analytics" {
+      where = $db.analytics.created_at >= $input.since
+      sort = { created_at: "asc" }
+      return = { type: "stream", chunk_size: 1000 }
+    } as $data
+
+    api.stream {
+      format = "jsonl"
+      value = $data
+    }
+  }
+}
+```
+
+---
+
+## Best Practices
+
+1. **Use streaming for large files** - Prevents memory exhaustion
+2. **Process in batches** - Commit database operations periodically
+3. **Handle errors gracefully** - Log failures without stopping stream
+4. **Set appropriate chunk sizes** - Balance memory and performance
+5. **Use JSONL for structured data** - Easier to parse than multi-line JSON

package/xanoscript_docs/syntax.md

@@ -382,3 +382,270 @@ conditional {
   }
 }
 ```
+
+---
+
+## System Variables
+
+Built-in variables available in the execution context.
+
+### Request Context
+
+| Variable | Description |
+|----------|-------------|
+| `$remote_ip` | Client IP address |
+| `$remote_port` | Client port number |
+| `$remote_host` | Remote hostname |
+| `$request_method` | HTTP method (GET, POST, etc.) |
+| `$request_uri` | Full request URI |
+| `$request_querystring` | Query string portion |
+| `$http_headers` | Request headers object |
+| `$request_auth_token` | Authorization token (if present) |
+
+### System Context
+
+| Variable | Description |
+|----------|-------------|
+| `$datasource` | Current data source name |
+| `$branch` | Current branch name |
+| `$tenant` | Tenant ID (multi-tenant apps) |
+| `$api_baseurl` | API base URL |
+| `$webflow` | Webflow context (if applicable) |
+
+### Access via $env
+
+```xs
+// Request variables accessed through $env
+var $client_ip { value = $env.$remote_ip }
+var $method { value = $env.$request_method }
+var $headers { value = $env.$http_headers }
+var $current_branch { value = $env.$branch }
+```
+
+---
+
+## Additional Operators
+
+### Nullish Coalescing
+
+Return right operand when left is null (not just falsy):
+
+```xs
+$value ?? "default"              // Returns "default" only if $value is null
+$value || "default"              // Returns "default" if $value is null, 0, "", or false
+```
+
+```xs
+// Difference example
+var $count { value = 0 }
+$count ?? 10                     // Returns 0 (not null)
+$count || 10                     // Returns 10 (0 is falsy)
+```
+
+### Database Filter Operators
+
+Additional operators for `db.query` where clauses:
+
+| Operator | Description | Example |
+|----------|-------------|---------|
+| `@>` | JSON contains | `$db.meta @> {"type": "featured"}` |
+| `~` | Regex match | `$db.name ~ "^test"` |
+| `!~` | Regex not match | `$db.name !~ "^draft"` |
+| `not in` | Not in list | `$db.status not in ["deleted", "hidden"]` |
+| `not between` | Not in range | `$db.price not between 0:10` |
+| `not contains` | Array not contains | `$db.tags not contains "spam"` |
+| `not includes` | String not includes | `$db.title not includes "test"` |
+| `not overlaps` | Arrays don't overlap | `$db.tags not overlaps ["hidden", "draft"]` |
+| `not ilike` | Case-insensitive not like | `$db.name not ilike "%test%"` |
+
+```xs
+db.query "product" {
+  where = $db.product.status not in ["deleted", "archived"]
+        && $db.product.metadata @> {"featured": true}
+        && $db.product.sku ~ "^SKU-[0-9]+"
+} as $products
+```
+
+---
+
+## Additional Filters
+
+### Text Domain Functions
+
+Functional equivalents for string operations:
+
+```xs
+text.contains("hello world", "world")        // true
+text.starts_with("hello", "he")              // true
+text.ends_with("hello", "lo")                // true
+text.icontains("Hello World", "WORLD")       // true (case-insensitive)
+text.istarts_with("Hello", "HE")             // true
+text.iends_with("Hello", "LO")               // true
+```
+
+### Object Domain Functions
+
+Functional equivalents for object operations:
+
+```xs
+object.keys({a: 1, b: 2})                    // ["a", "b"]
+object.values({a: 1, b: 2})                  // [1, 2]
+object.entries({a: 1, b: 2})                 // [{key: "a", value: 1}, {key: "b", value: 2}]
+```
+
+### Math Domain Functions
+
+Functional equivalents for math operations:
+
+```xs
+math.add(5, 3)                               // 8
+math.sub(10, 4)                              // 6
+math.mul(3, 4)                               // 12
+math.div(20, 5)                              // 4
+math.mod(10, 3)                              // 1
+```
+
+### Bitwise Operations
+
+```xs
+// As filters
+5|bitwise_and:3                              // 1
+5|bitwise_or:3                               // 7
+5|bitwise_xor:3                              // 6
+5|bitwise_not                                // -6
+
+// As functions
+math.bitwise.and(5, 3)                       // 1
+math.bitwise.or(5, 3)                        // 7
+math.bitwise.xor(5, 3)                       // 6
+```
+
+### Logical NOT Filter
+
+```xs
+true|not                                     // false
+false|not                                    // true
+$condition|not                               // Inverts boolean
+```
+
+### Array Filtering
+
+| Filter | Description | Example |
+|--------|-------------|---------|
+| `filter_empty` | Remove empty values | `$arr\|filter_empty` |
+| `filter_empty_text` | Remove empty strings | `$arr\|filter_empty_text` |
+| `filter_empty_array` | Remove empty arrays | `$arr\|filter_empty_array` |
+| `filter_empty_object` | Remove empty objects | `$arr\|filter_empty_object` |
+| `filter_null` | Remove null values | `$arr\|filter_null` |
+| `filter_zero` | Remove zero values | `$arr\|filter_zero` |
+| `filter_false` | Remove false values | `$arr\|filter_false` |
+
+```xs
+[1, null, "", 0, "text", false]|filter_empty      // [1, "text"]
+["a", "", "b", ""]|filter_empty_text              // ["a", "b"]
+```
+
+### Array Fill Operations
+
+```xs
+|fill:5:"x"                                  // ["x", "x", "x", "x", "x"]
+["a", "b"]|fill_keys:{"a": 1, "b": 2}        // {a: 1, b: 2}
+```
+
+### Deep Merge & Comparison
+
+```xs
+{a: {b: 1}}|merge_recursive:{a: {c: 2}}      // {a: {b: 1, c: 2}}
+[1, 2, 3]|diff_assoc:[2]                     // Associative diff
+[1, 2, 3]|intersect_assoc:[2, 3, 4]          // Associative intersect
+```
+
+### Encoding Filters
+
+| Filter | Description | Example |
+|--------|-------------|---------|
+| `list_encodings` | List available encodings | `\|list_encodings` |
+| `detect_encoding` | Detect string encoding | `$text\|detect_encoding` |
+| `to_utf8` | Convert to UTF-8 | `$text\|to_utf8` |
+| `from_utf8` | Convert from UTF-8 | `$text\|from_utf8:"ISO-8859-1"` |
+| `convert_encoding` | Convert encodings | `$text\|convert_encoding:"UTF-8":"ISO-8859-1"` |
+
+```xs
+// CSV parsing (alternative to csv_decode)
+$csv_text|csv_parse                          // Parse CSV string
+$data|csv_create                             // Create CSV string
+
+// Query string
+"a=1&b=2"|querystring_parse                  // {a: "1", b: "2"}
+```
+
+### String Escape Filters
+
+| Filter | Description |
+|--------|-------------|
+| `addslashes` | Escape quotes and backslashes |
+| `escape` | HTML escape |
+| `text_escape` | Escape for text output |
+| `text_unescape` | Unescape text |
+| `regex_quote` | Escape regex special characters |
+
+```xs
+"Hello \"World\""|addslashes                 // "Hello \\\"World\\\""
+"<script>"|escape                            // "&lt;script&gt;"
+"^test$"|regex_quote                         // "\^test\$"
+```
+
+### Trigonometry Examples
+
+```xs
+// Radians and degrees
+90|deg2rad                                   // 1.5707963...
+1.5707963|rad2deg                            // 90
+
+// Trig functions (input in radians)
+0|sin                                        // 0
+0|cos                                        // 1
+0.785398|tan                                 // ~1 (45 degrees)
+
+// Inverse trig
+0|asin                                       // 0
+1|acos                                       // 0
+1|atan                                       // 0.785398...
+
+// Hyperbolic
+0|sinh                                       // 0
+0|cosh                                       // 1
+0|tanh                                       // 0
+```
+
+### DB Query Timestamp Filters
+
+Extended timestamp operations for database queries:
+
+```xs
+$db.created_at|timestamp_year                // Extract year
+$db.created_at|timestamp_month               // Extract month (1-12)
+$db.created_at|timestamp_week                // Extract week number
+$db.created_at|timestamp_day_of_month        // Day of month (1-31)
+$db.created_at|timestamp_day_of_week         // Day of week (0-6)
+$db.created_at|timestamp_day_of_year         // Day of year (1-366)
+$db.created_at|timestamp_hour                // Hour (0-23)
+$db.created_at|timestamp_minute              // Minute (0-59)
+
+// Epoch variants
+$db.created_at|timestamp_epoch_seconds       // Seconds since epoch
+$db.created_at|timestamp_epoch_ms            // Milliseconds since epoch
+```
+
+### Vector Operations (AI/ML)
+
+Additional vector similarity functions:
+
+```xs
+$db.embedding|l1_distance_manhattan:$input.vector    // L1/Manhattan distance
+$db.embedding|negative_inner_product:$input.vector   // Negative inner product
+$db.embedding|inner_product:$input.vector            // Inner product
+
+// Geo covers (for polygon containment)
+$db.boundary|covers:$input.point                     // Polygon covers point
+```

package/xanoscript_docs/types.md

@@ -176,35 +176,80 @@ geo_multipoint points
 Filters validate and transform input values. Chain with `|`.
 
 ### String Filters
-| Filter | Description |
-|--------|-------------|
-| `trim` | Remove leading/trailing whitespace |
-| `lower` | Convert to lowercase |
-| `upper` | Convert to uppercase |
-| `min:<n>` | Minimum length |
-| `max:<n>` | Maximum length |
-| `ok:<chars>` | Allow only specified characters |
-| `prevent:<str>` | Block specific substrings |
-| `startsWith:<prefix>` | Require prefix |
-| `alphaOk` | Allow only letters |
-| `digitOk` | Allow only digits |
+| Filter | Description | Example |
+|--------|-------------|---------|
+| `trim` | Remove leading/trailing whitespace | `text name filters=trim` |
+| `lower` | Convert to lowercase | `text email filters=lower` |
+| `upper` | Convert to uppercase | `text code filters=upper` |
+| `min:<n>` | Minimum length | `text name filters=min:2` |
+| `max:<n>` | Maximum length | `text bio filters=max:500` |
+| `ok:<chars>` | Allow only specified characters | `text hex filters=ok:0123456789abcdef` |
+| `prevent:<str>` | Block specific substrings | `text name filters=prevent:admin` |
+| `startsWith:<prefix>` | Require prefix | `text sku filters=startsWith:SKU-` |
+| `endsWith:<suffix>` | Require suffix | `text file filters=endsWith:.pdf` |
+| `alphaOk` | Allow only letters (a-zA-Z) | `text name filters=alphaOk` |
+| `digitOk` | Allow only digits (0-9) | `text code filters=digitOk` |
+| `alphaNumOk` | Allow letters and digits | `text username filters=alphaNumOk` |
+| `pattern:<regex>` | Match regex pattern | `text phone filters=pattern:^\+?[0-9]+$` |
 
 ### Numeric Filters
-| Filter | Description |
-|--------|-------------|
-| `min:<n>` | Minimum value |
-| `max:<n>` | Maximum value |
+| Filter | Description | Example |
+|--------|-------------|---------|
+| `min:<n>` | Minimum value | `int age filters=min:0` |
+| `max:<n>` | Maximum value | `int age filters=max:150` |
+| `between:<a>:<b>` | Value between a and b | `int score filters=between:0:100` |
+
+### Array Filters
+| Filter | Description | Example |
+|--------|-------------|---------|
+| `min:<n>` | Minimum array length | `text[] tags filters=min:1` |
+| `max:<n>` | Maximum array length | `text[] tags filters=max:10` |
+| `unique` | Remove duplicates | `int[] ids filters=unique` |
+
+### Character Set Filters
 
-### Examples
 ```xs
 input {
-  text username filters=trim|lower|min:3|max:20|alphaOk
+  // Allow only specific characters
+  text hex_code filters=ok:0123456789abcdef
+
+  // Allow letters only
+  text name filters=alphaOk
+
+  // Allow digits only
+  text pin filters=digitOk
+
+  // Combine: letters, digits, and underscore
+  text username filters=ok:abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_
+}
+```
+
+### Pattern Validation
+
+```xs
+input {
+  // Phone number pattern
+  text phone filters=pattern:^\+?[1-9]\d{1,14}$
+
+  // Postal code pattern
+  text zip filters=pattern:^\d{5}(-\d{4})?$
+
+  // Custom ID format
+  text product_id filters=pattern:^[A-Z]{2}-\d{4}$
+}
+```
+
+### Combined Examples
+```xs
+input {
+  text username filters=trim|lower|min:3|max:20|alphaNumOk
   email contact filters=trim|lower
   int age filters=min:0|max:150
-  text hex_code filters=ok:abcdef0123456789
-  text[] tags filters=trim|lower|max:50
+  text hex_code filters=ok:abcdef0123456789|min:6|max:6
+  text[] tags filters=trim|lower|max:50|unique
+  text phone filters=trim|pattern:^\+?[0-9\s-]+$
+  int quantity filters=min:1|max:100
 }
-```
 
 ---