@polyglot-sql/sdk 0.1.9 → 0.1.10

package/dist/index.d.ts

@@ -1524,6 +1524,10 @@ declare type Column = {
      * Trailing comments that appeared after this column reference
      */
     trailing_comments: Array<string>;
+    /**
+     * Source position span
+     */
+    span?: Span | null;
 };
 
 /**
@@ -3131,6 +3135,7 @@ declare const _default: {
     isInitialized: typeof isInitialized;
     transpile: typeof transpile;
     parse: typeof parse;
+    tokenize: typeof tokenize;
     generate: typeof generate;
     format: typeof format;
     getDialects: typeof getDialects;
@@ -6200,6 +6205,10 @@ declare type Function_2 = {
      * Whether the function name was quoted (e.g., `p.d.UdF` in BigQuery)
      */
     quoted: boolean;
+    /**
+     * Source position span
+     */
+    span?: Span | null;
 };
 
 /**
@@ -6862,6 +6871,10 @@ declare type Identifier = {
      */
     quoted: boolean;
     trailing_comments: Array<string>;
+    /**
+     * Source position span (populated during parsing, None for programmatically constructed nodes)
+     */
+    span?: Span | null;
 };
 
 /**
@@ -9730,6 +9743,10 @@ export declare interface ParseResult {
     errorLine?: number;
     /** 1-based column number where the error occurred */
     errorColumn?: number;
+    /** Start byte offset of the error range */
+    errorStart?: number;
+    /** End byte offset of the error range (exclusive) */
+    errorEnd?: number;
 }
 
 /**
@@ -10049,6 +10066,10 @@ export declare class Polyglot {
      * Parse SQL into an AST.
      */
     parse(sql: string, dialect?: Dialect): ParseResult;
+    /**
+     * Tokenize SQL into a token stream.
+     */
+    tokenize(sql: string, dialect?: Dialect): TokenizeResult;
     /**
      * Generate SQL from an AST.
      */
@@ -11715,6 +11736,38 @@ export declare interface SourceTablesResult {
     error?: string;
 }
 
+/**
+ * Represents a position in the source SQL
+ */
+declare type Span = {
+    /**
+     * Starting byte offset
+     */
+    start: number;
+    /**
+     * Ending byte offset (exclusive)
+     */
+    end: number;
+    /**
+     * Line number (1-based)
+     */
+    line: number;
+    /**
+     * Column number (1-based)
+     */
+    column: number;
+};
+
+/**
+ * Span information for a token, indicating its position in the source SQL.
+ */
+export declare interface SpanInfo {
+    start: number;
+    end: number;
+    line: number;
+    column: number;
+}
+
 /**
  * SPLIT function
  */
@@ -11842,6 +11895,10 @@ declare type Star = {
      * Trailing comments that appeared after the star
      */
     trailing_comments?: Array<string>;
+    /**
+     * Source position span
+     */
+    span?: Span | null;
 };
 
 /** Create a star (*) expression. */
@@ -12421,6 +12478,10 @@ declare type TableRef = {
      * Time travel version clause: FOR VERSION AS OF / FOR TIMESTAMP AS OF (Presto/Trino, BigQuery, Databricks)
      */
     version?: Version | null;
+    /**
+     * Source position span
+     */
+    span?: Span | null;
 };
 
 /**
@@ -12711,6 +12772,53 @@ declare type ToFile = {
     safe: Expression | null;
 };
 
+/**
+ * A single token from the SQL token stream.
+ */
+export declare interface TokenInfo {
+    tokenType: string;
+    text: string;
+    span: SpanInfo;
+    comments: string[];
+    trailingComments: string[];
+}
+
+/**
+ * Tokenize SQL into a token stream.
+ *
+ * @param sql - The SQL string to tokenize
+ * @param dialect - The dialect to use for tokenization
+ * @returns The token stream
+ *
+ * @example
+ * ```typescript
+ * const result = tokenize("SELECT a, b FROM t", Dialect.PostgreSQL);
+ * if (result.success) {
+ *   for (const token of result.tokens!) {
+ *     console.log(token.tokenType, token.text, token.span);
+ *   }
+ * }
+ * ```
+ */
+export declare function tokenize(sql: string, dialect?: Dialect): TokenizeResult;
+
+/**
+ * Result of a tokenize operation
+ */
+export declare interface TokenizeResult {
+    success: boolean;
+    tokens?: TokenInfo[];
+    error?: string;
+    /** 1-based line number where the error occurred */
+    errorLine?: number;
+    /** 1-based column number where the error occurred */
+    errorColumn?: number;
+    /** Start byte offset of the error range */
+    errorStart?: number;
+    /** End byte offset of the error range (exclusive) */
+    errorEnd?: number;
+}
+
 /**
  * ToMap - Materialize-style map constructor
  * Can hold either:
@@ -12899,6 +13007,10 @@ export declare interface TranspileResult {
     errorLine?: number;
     /** 1-based column number where the error occurred */
     errorColumn?: number;
+    /** Start byte offset of the error range */
+    errorStart?: number;
+    /** End byte offset of the error range (exclusive) */
+    errorEnd?: number;
 }
 
 /**
@@ -13586,6 +13698,10 @@ export declare interface ValidationError {
     severity: ValidationSeverity;
     /** Error code (e.g., "E001", "W001") */
     code: string;
+    /** Start byte offset of the error range */
+    start?: number;
+    /** End byte offset of the error range (exclusive) */
+    end?: number;
 }
 
 /**

package/dist/index.js

@@ -2342,6 +2342,55 @@ function source_tables$1(sql, column, dialect) {
     }
 }
 
+/**
+ * Tokenize SQL into a token stream (returned as JSON).
+ *
+ * # Arguments
+ * * `sql` - The SQL string to tokenize
+ * * `dialect` - The dialect to use for tokenization
+ *
+ * # Returns
+ * A JSON string containing the TokenizeResult
+ * @param {string} sql
+ * @param {string} dialect
+ * @returns {string}
+ */
+function tokenize$2(sql, dialect) {
+    let deferred3_0;
+    let deferred3_1;
+    try {
+        const retptr = wasm$3.__wbindgen_add_to_stack_pointer(-16);
+        const ptr0 = passStringToWasm0(sql, wasm$3.__wbindgen_export, wasm$3.__wbindgen_export2);
+        const len0 = WASM_VECTOR_LEN;
+        const ptr1 = passStringToWasm0(dialect, wasm$3.__wbindgen_export, wasm$3.__wbindgen_export2);
+        const len1 = WASM_VECTOR_LEN;
+        wasm$3.tokenize(retptr, ptr0, len0, ptr1, len1);
+        var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+        var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+        deferred3_0 = r0;
+        deferred3_1 = r1;
+        return getStringFromWasm0(r0, r1);
+    } finally {
+        wasm$3.__wbindgen_add_to_stack_pointer(16);
+        wasm$3.__wbindgen_export4(deferred3_0, deferred3_1, 1);
+    }
+}
+
+/**
+ * Tokenize SQL and return a structured JS object.
+ * @param {string} sql
+ * @param {string} dialect
+ * @returns {any}
+ */
+function tokenize_value$1(sql, dialect) {
+    const ptr0 = passStringToWasm0(sql, wasm$3.__wbindgen_export, wasm$3.__wbindgen_export2);
+    const len0 = WASM_VECTOR_LEN;
+    const ptr1 = passStringToWasm0(dialect, wasm$3.__wbindgen_export, wasm$3.__wbindgen_export2);
+    const len1 = WASM_VECTOR_LEN;
+    const ret = wasm$3.tokenize_value(ptr0, len0, ptr1, len1);
+    return takeObject(ret);
+}
+
 /**
  * Transpile SQL from one dialect to another.
  *
@@ -3359,6 +3408,8 @@ const parse$1 = __vite__wasmModule.parse;
 const parse_value = __vite__wasmModule.parse_value;
 const plan$1 = __vite__wasmModule.plan;
 const source_tables = __vite__wasmModule.source_tables;
+const tokenize$1 = __vite__wasmModule.tokenize;
+const tokenize_value = __vite__wasmModule.tokenize_value;
 const transpile$1 = __vite__wasmModule.transpile;
 const transpile_value = __vite__wasmModule.transpile_value;
 const validate$1 = __vite__wasmModule.validate;
@@ -3544,6 +3595,8 @@ const wasm$2 = /*#__PURE__*/Object.freeze(/*#__PURE__*/Object.defineProperty({
     parse_value,
     plan: plan$1,
     source_tables,
+    tokenize: tokenize$1,
+    tokenize_value,
     transpile: transpile$1,
     transpile_value,
     validate: validate$1,
@@ -3723,6 +3776,8 @@ const wasmModule = /*#__PURE__*/Object.freeze(/*#__PURE__*/Object.defineProperty
     parse_value: parse_value$1,
     plan: plan$2,
     source_tables: source_tables$1,
+    tokenize: tokenize$2,
+    tokenize_value: tokenize_value$1,
     transpile: transpile$2,
     transpile_value: transpile_value$1,
     validate: validate$2,
@@ -5186,10 +5241,16 @@ function validateSelect(select, _options) {
   const errors = [];
   const stars = findByType(select, "star");
   if (stars.length > 0) {
+    const starData = getExprData(stars[0]);
+    const span = starData?.span;
     errors.push({
       message: "SELECT * is discouraged; specify columns explicitly for better performance and maintainability",
       severity: "warning",
-      code: "W001"
+      code: "W001",
+      line: span?.line,
+      column: span?.column,
+      start: span?.start,
+      end: span?.end
     });
   }
   const selectData = getExprData(select);
@@ -5205,10 +5266,22 @@ function validateSelect(select, _options) {
       return false;
     });
     if (hasNonAggregateColumn && aggregateFunctions.length > 0) {
+      const firstNonAgg = selectCols.find((col) => {
+        const colType = getExprType(col);
+        if (colType === "column" || colType === "identifier") {
+          return getAggregateFunctions(col).length === 0;
+        }
+        return false;
+      });
+      const colSpan = firstNonAgg ? getExprData(firstNonAgg)?.span : void 0;
       errors.push({
         message: "Mixing aggregate functions with non-aggregated columns without GROUP BY may cause errors in strict SQL mode",
         severity: "warning",
-        code: "W002"
+        code: "W002",
+        line: colSpan?.line,
+        column: colSpan?.column,
+        start: colSpan?.start,
+        end: colSpan?.end
       });
     }
   }
@@ -5220,10 +5293,16 @@ function validateSelect(select, _options) {
     });
   }
   if (selectData.limit && !selectData.order_by) {
+    const limitExpr = selectData.limit;
+    const limitSpan = limitExpr ? getExprData(limitExpr)?.span : void 0;
     errors.push({
       message: "LIMIT without ORDER BY produces non-deterministic results",
       severity: "warning",
-      code: "W004"
+      code: "W004",
+      line: limitSpan?.line,
+      column: limitSpan?.column,
+      start: limitSpan?.start,
+      end: limitSpan?.end
     });
   }
   return errors;
@@ -5435,6 +5514,22 @@ function parse(sql, dialect = "generic" /* Generic */) {
     return parseFailure("parse", error);
   }
 }
+function tokenize(sql, dialect = "generic" /* Generic */) {
+  try {
+    if (typeof wasm.tokenize_value === "function") {
+      return decodeWasmPayload(
+        wasm.tokenize_value(sql, dialect)
+      );
+    }
+    return JSON.parse(wasm.tokenize(sql, dialect));
+  } catch (error) {
+    return {
+      success: false,
+      tokens: void 0,
+      error: `WASM tokenize failed: ${errorMessage(error)}`
+    };
+  }
+}
 function generate(ast2, dialect = "generic" /* Generic */) {
   try {
     if (typeof wasm.generate_value === "function" && Array.isArray(ast2)) {
@@ -5512,6 +5607,12 @@ class Polyglot {
   parse(sql, dialect = "generic" /* Generic */) {
     return parse(sql, dialect);
   }
+  /**
+   * Tokenize SQL into a token stream.
+   */
+  tokenize(sql, dialect = "generic" /* Generic */) {
+    return tokenize(sql, dialect);
+  }
   /**
    * Generate SQL from an AST.
    */
@@ -5549,6 +5650,7 @@ const index = {
   isInitialized,
   transpile,
   parse,
+  tokenize,
   generate,
   format,
   getDialects,
@@ -5563,4 +5665,4 @@ const index = {
   Polyglot
 };
 
-export { CaseBuilder, DeleteBuilder, Dialect, Expr, InsertBuilder, MergeBuilder, Polyglot, SelectBuilder, SetOpBuilder, UpdateBuilder, ValidationSeverity, WindowDefBuilder, abs, alias, and, index$1 as ast, avg, boolean, caseOf, caseWhen, cast, ceil, changesOnly, coalesce, col, concatWs, condition, count, countDistinct, currentDate, currentTime, currentTimestamp, index as default, del, deleteFrom, denseRank, diff, except, exp, extract, findAll, floor, format, formatWithOptions, func, generate, getColumns, getDialects, getSourceTables, getVersion, greatest, hasChanges, ifNull, init, initcap, insert, insertInto, intersect, isColumn, isFunction, isInitialized, isLiteral, isSelect, least, length, lineage, lit, ln, lower, ltrim, max, mergeInto, min, not, nullIf, or, parse, plan, power, rank, renameColumns, replace, reverse, round, rowNumber, rtrim, select, sign, sqlExpr, sqlNull, sqrt, star, subquery, substring, sum, table, transform, transpile, trim, union, unionAll, update, upper, validate, validateWithSchema, walk };
+export { CaseBuilder, DeleteBuilder, Dialect, Expr, InsertBuilder, MergeBuilder, Polyglot, SelectBuilder, SetOpBuilder, UpdateBuilder, ValidationSeverity, WindowDefBuilder, abs, alias, and, index$1 as ast, avg, boolean, caseOf, caseWhen, cast, ceil, changesOnly, coalesce, col, concatWs, condition, count, countDistinct, currentDate, currentTime, currentTimestamp, index as default, del, deleteFrom, denseRank, diff, except, exp, extract, findAll, floor, format, formatWithOptions, func, generate, getColumns, getDialects, getSourceTables, getVersion, greatest, hasChanges, ifNull, init, initcap, insert, insertInto, intersect, isColumn, isFunction, isInitialized, isLiteral, isSelect, least, length, lineage, lit, ln, lower, ltrim, max, mergeInto, min, not, nullIf, or, parse, plan, power, rank, renameColumns, replace, reverse, round, rowNumber, rtrim, select, sign, sqlExpr, sqlNull, sqrt, star, subquery, substring, sum, table, tokenize, transform, transpile, trim, union, unionAll, update, upper, validate, validateWithSchema, walk };

package/package.json

@@ -1,17 +1,21 @@
 {
   "name": "@polyglot-sql/sdk",
-  "version": "0.1.9",
+  "version": "0.1.10",
   "description": "SQL dialect translator powered by WebAssembly",
   "type": "module",
-  "main": "./dist/index.js",
+  "main": "./dist/index.cjs",
   "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "unpkg": "./dist/cdn/polyglot.esm.js",
   "jsdelivr": "./dist/cdn/polyglot.esm.js",
   "exports": {
     ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js"
+      "types": {
+        "import": "./dist/index.d.ts",
+        "require": "./dist/index.d.cts"
+      },
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
     },
     "./cdn": {
       "types": "./dist/index.d.ts",
@@ -62,10 +66,11 @@
     "build:bindings": "cd ../.. && make generate-bindings",
     "build:wasm": "cd ../../crates/polyglot-sql-wasm && wasm-pack build --target bundler --release --out-dir ../../packages/sdk/wasm",
     "build:esm": "vite build",
+    "build:cjs": "vite build --config vite.config.cjs.ts && cp dist/index.d.ts dist/index.d.cts",
     "build:umd": "vite build --config vite.config.umd.ts",
     "build:dialect": "vite build --config vite.config.dialect.ts",
     "build:dialects": "for d in postgresql mysql bigquery snowflake duckdb tsql clickhouse; do POLYGLOT_DIALECT=$d pnpm run build:dialect; done",
-    "build": "pnpm run build:esm && pnpm run build:umd",
+    "build": "pnpm run build:esm && pnpm run build:cjs && pnpm run build:umd",
     "build:all": "pnpm run build && pnpm run build:dialects",
     "test": "vitest run",
     "test:watch": "vitest",