@polyglot-sql/sdk 0.1.8 → 0.1.10

package/README.md

@@ -50,6 +50,38 @@ console.log(sql[0]);
 //   x = 1
 ```
 
+### Format With Guard Options
+
+Formatting guard defaults from Rust core:
+- `maxInputBytes`: `16 * 1024 * 1024`
+- `maxTokens`: `1_000_000`
+- `maxAstNodes`: `1_000_000`
+- `maxSetOpChain`: `256`
+
+```typescript
+import { formatWithOptions, Dialect } from '@polyglot-sql/sdk';
+
+const result = formatWithOptions(
+  'SELECT a,b FROM t WHERE x=1',
+  Dialect.PostgreSQL,
+  {
+    maxInputBytes: 2 * 1024 * 1024,
+    maxTokens: 250_000,
+    maxAstNodes: 250_000,
+    maxSetOpChain: 128,
+  },
+);
+
+if (!result.success) {
+  // Includes one of:
+  // E_GUARD_INPUT_TOO_LARGE
+  // E_GUARD_TOKEN_BUDGET_EXCEEDED
+  // E_GUARD_AST_BUDGET_EXCEEDED
+  // E_GUARD_SET_OP_CHAIN_EXCEEDED
+  console.error(result.error);
+}
+```
+
 ## Fluent Query Builder
 
 Build SQL queries programmatically with full type safety. All builder operations are backed by the Rust engine via WASM.
@@ -430,9 +462,37 @@ const cartesian = validateWithSchema(
 );
 ```
 
+## Tokenize
+
+Access the raw SQL token stream with full source position spans. Useful for syntax highlighting, custom linters, or editor integrations.
+
+```typescript
+import { tokenize, Dialect } from '@polyglot-sql/sdk';
+
+const result = tokenize('SELECT a, b FROM t', Dialect.Generic);
+if (result.success) {
+  for (const token of result.tokens!) {
+    console.log(token.tokenType, token.text, token.span);
+    // "Select" "SELECT" { start: 0, end: 6, line: 1, column: 1 }
+    // "Var"    "a"      { start: 7, end: 8, line: 1, column: 8 }
+    // ...
+  }
+}
+```
+
+Each token includes:
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `tokenType` | `string` | Token type name (e.g. `"Select"`, `"Var"`, `"Comma"`) |
+| `text` | `string` | Raw source text of the token |
+| `span` | `SpanInfo` | Source position: `start`/`end` byte offsets, `line`/`column` (1-based) |
+| `comments` | `string[]` | Leading comments attached to this token |
+| `trailingComments` | `string[]` | Trailing comments attached to this token |
+
 ## Error Reporting
 
-Parse and transpile errors include source position information, making it easy to highlight errors in editors or show precise error messages.
+Parse, transpile, and tokenize errors include source position information with both line/column and byte offset ranges, making it easy to highlight errors in editors or show precise error messages.
 
 ```typescript
 import { parse, transpile, Dialect } from '@polyglot-sql/sdk';
@@ -442,15 +502,19 @@ if (!result.success) {
   console.log(result.error);       // "Parse error at line 1, column 11: ..."
   console.log(result.errorLine);   // 1
   console.log(result.errorColumn); // 11
+  console.log(result.errorStart);  // 10 (byte offset)
+  console.log(result.errorEnd);    // 11 (byte offset, exclusive)
 }
 ```
 
-Both `ParseResult` and `TranspileResult` include optional position fields:
+`ParseResult`, `TranspileResult`, and `TokenizeResult` include optional position fields:
 
 | Field | Type | Description |
 |-------|------|-------------|
 | `errorLine` | `number \| undefined` | 1-based line number where the error occurred |
 | `errorColumn` | `number \| undefined` | 1-based column number where the error occurred |
+| `errorStart` | `number \| undefined` | Start byte offset of the error range (0-based) |
+| `errorEnd` | `number \| undefined` | End byte offset of the error range (exclusive) |
 
 These fields are only present when `success` is `false`. On success, they are `undefined`.
 
@@ -533,6 +597,10 @@ import { Polyglot, Dialect } from '@polyglot-sql/sdk';
 const pg = Polyglot.getInstance();
 const result = pg.transpile('SELECT 1', Dialect.MySQL, Dialect.PostgreSQL);
 const formatted = pg.format('SELECT a,b FROM t');
+const formattedSafe = pg.formatWithOptions('SELECT a,b FROM t', Dialect.Generic, {
+  maxInputBytes: 2 * 1024 * 1024,
+  maxSetOpChain: 128,
+});
 ```
 
 ## API Reference
@@ -545,6 +613,8 @@ const formatted = pg.format('SELECT a,b FROM t');
 | `parse(sql, dialect?)` | Parse SQL into AST |
 | `generate(ast, dialect?)` | Generate SQL from AST |
 | `format(sql, dialect?)` | Pretty-print SQL |
+| `formatWithOptions(sql, dialect?, options?)` | Pretty-print SQL with guard overrides |
+| `tokenize(sql, dialect?)` | Tokenize SQL into a token stream with source spans |
 | `validate(sql, dialect?, options?)` | Validate SQL syntax/semantics |
 | `validateWithSchema(sql, schema, dialect?, options?)` | Validate against a database schema |
 | `getDialects()` | List supported dialect names |
@@ -696,6 +766,44 @@ For browser use without a bundler:
 </script>
 ```
 
+## CommonJS (CJS) Usage
+
+For Node.js projects using `require()`, the SDK ships a CJS build. Since WASM cannot be loaded synchronously, you must call `init()` before using any other function:
+
+```javascript
+const { init, transpile, parse, select, col, lit, isInitialized } = require('@polyglot-sql/sdk');
+
+async function main() {
+  await init();
+
+  // Now all functions work
+  const result = transpile('SELECT IFNULL(a, b)', 'mysql', 'postgresql');
+  console.log(result.sql[0]); // SELECT COALESCE(a, b)
+
+  const parsed = parse('SELECT 1', 'generic');
+  console.log(parsed.success); // true
+
+  const sql = select('id', 'name').from('users')
+    .where(col('id').eq(lit(1)))
+    .toSql();
+  console.log(sql); // SELECT id, name FROM users WHERE id = 1
+}
+
+main();
+```
+
+You can check initialization status with `isInitialized()`:
+
+```javascript
+const { init, isInitialized } = require('@polyglot-sql/sdk');
+
+console.log(isInitialized()); // false
+await init();
+console.log(isInitialized()); // true
+```
+
+> **Note:** The ESM build (`import`) auto-initializes via top-level `await`, so `init()` is not required there. The CJS build requires it because `require()` is synchronous.
+
 ## License
 
 [MIT](../../LICENSE)