finox 0.1.0 → 0.2.0

data/README.md

@@ -2,44 +2,49 @@
 
 A MySQL query parser for Ruby, powered by [sqlparser-rs](https://github.com/apache/datafusion-sqlparser-rs).
 
-Finox parses SQL with Rust and returns the AST as plain Ruby Hashes and Arrays.
+Finox parses SQL with Rust and exposes tables, columns, fingerprints and the raw AST.
 
 ## Installation
 
-Add to your Gemfile:
-
 ```bash
 bundle add finox
 ```
 
-Or install directly:
+Or without Bundler:
 
 ```bash
 gem install finox
 ```
 
-A Rust toolchain is required to build the native extension.
+Precompiled native gems are available for Ruby 3.1–4.0 on the following
+platforms, so installation there requires no Rust toolchain:
+
+| OS      | Platforms                                        |
+| ------- | ------------------------------------------------ |
+| Linux   | x86_64, aarch64 (glibc and musl variants)        |
+| macOS   | x86_64, arm64                                    |
+| Windows | x64-mingw-ucrt                                   |
+
+On any other platform the source gem is installed instead, and compiling it
+requires a Rust toolchain.
 
 ## Usage
 
+`Finox.parse` returns a `Finox::Result`:
+
 ```ruby
 require "finox"
 
-ast = Finox.parse("SELECT `name` FROM `users` WHERE id = 1")
-# => [{"Query" => {with: nil, body: {"Select" => {...}}, ...}}]
+result = Finox.parse("SELECT `name` FROM `users` WHERE id = 1")
+# => #<Finox::Result>
 
-ast.dig(0, "Query", :body, "Select", :projection)
-# => [{"UnnamedExpr" =>
-#       {"Identifier" =>
-#         {value: "name",
-#          quote_style: "`",
-#          span: {start: {line: 1, column: 8}, end: {line: 1, column: 14}}}}}]
+result.tables          # => ["users"]
+result.columns         # => ["name", "id"]
+result.statement_types # => ["Query"]
+result.normalize       # => "SELECT `name` FROM `users` WHERE id = ?"
+result.fingerprint     # => "24e307ca0f02abfc"
 ```
 
-`Finox.parse` returns one Hash per statement, so `"SELECT 1; SELECT 2"` yields an array of two.
-
-Key convention (from serde's externally tagged enums): enum variant names are `String` keys (`"Query"`, `"Select"`, `"Identifier"`), struct fields are `Symbol` keys (`:body`, `:projection`, `:value`).
-
 Invalid SQL raises `Finox::ParseError`:
 
 ```ruby
@@ -47,6 +52,69 @@ Finox.parse("SELEKT 1")
 # => Finox::ParseError: sql parser error: Expected: an SQL statement, found: SELEKT at Line: 1, Column: 1
 ```
 
+### Table classification
+
+`#select_tables`, `#dml_tables` and `#ddl_tables` classify the referenced
+tables by how they are used: read, written by DML (`INSERT` / `UPDATE` /
+`DELETE`) or targeted by DDL (`CREATE TABLE` / `ALTER TABLE` / `DROP TABLE` /
+`TRUNCATE` etc.). A table appearing in multiple roles is listed in each.
+
+```ruby
+result = Finox.parse("INSERT INTO logs SELECT message FROM events; DROP TABLE archives")
+
+result.tables        # => ["logs", "events", "archives"]
+result.select_tables # => ["events"]
+result.dml_tables    # => ["logs"]
+result.ddl_tables    # => ["archives"]
+```
+
+#### Known limitations
+
+Extraction is syntactic, with no schema knowledge and only coarse scope
+resolution:
+
+- A CTE name shadows a same-named real table everywhere in the statement,
+  even inside the CTE's own definition.
+- Multi-table `UPDATE` (`UPDATE t1 JOIN t2 ... SET t2.x = 1`) reports only
+  `t1` in `#dml_tables`.
+- `#columns` does not resolve table aliases (`u.id` stays `u.id`).
+
+### Normalization and fingerprints
+
+`#normalize` replaces literals with `?` placeholders and deparses from the
+AST, so formatting and keyword case are normalized as well. `#fingerprint` is
+a 64-bit hex hash (xxhash64) of the normalized SQL — stable across literal
+and formatting differences, but not guaranteed to be stable across finox
+versions, since the normalized form depends on the bundled sqlparser-rs.
+Recompute stored fingerprints when upgrading finox.
+
+```ruby
+Finox.parse("select * from users where id=1").normalize
+# => "SELECT * FROM users WHERE id = ?"
+
+Finox.parse("select * from users where id=1").fingerprint ==
+  Finox.parse("SELECT * FROM users WHERE id = 42").fingerprint
+# => true
+```
+
+### Raw sqlparser-rs AST
+
+For anything not covered above, `#statements` exposes sqlparser-rs's raw AST of each parsed statement as plain
+Ruby Hashes and Arrays:
+
+```ruby
+ast = Finox.parse("SELECT `name` FROM `users` WHERE id = 1").statements.first
+# => {"Query" => {"with" => nil, "body" => {"Select" => {...}}, ...}}
+
+ast.dig("Query", "body", "Select", "projection")
+# => [{"UnnamedExpr" =>
+#       {"Identifier" =>
+#         {"value" => "name",
+#          "quote_style" => "`",
+#          "span" => {"start" => {"line" => 1, "column" => 8},
+#                     "end" => {"line" => 1, "column" => 14}}}}}]
+```
+
 ## Development
 
 ```bash
@@ -58,3 +126,7 @@ bundle exec rake          # compile + spec + rubocop
 ## License
 
 [MIT](LICENSE.txt)
+
+Precompiled gems statically link Rust crates, including sqlparser-rs
+(Apache-2.0). See [LICENSE-THIRD-PARTY.txt](LICENSE-THIRD-PARTY.txt) for
+their licenses and attributions.

data/ext/finox/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "finox"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 authors = ["kyuuri1791"]
 license = "MIT"
@@ -11,5 +11,6 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = "0.8"
-serde_magnus = "0.11.0"
-sqlparser = { version = "0.62.0", features = ["serde"] }
+serde_json = { version = "1", features = ["preserve_order"] }
+sqlparser = { version = "0.62.0", features = ["serde", "visitor"] }
+twox-hash = { version = "2.1", default-features = false, features = ["xxhash64"] }

data/ext/finox/src/lib.rs

@@ -1,5 +1,16 @@
-use magnus::{function, prelude::*, value::Lazy, Error, ExceptionClass, Ruby, Value};
+use std::collections::HashSet;
+use std::fmt::{self, Write as _};
+use std::ops::ControlFlow;
+
+use magnus::{
+    function, method, prelude::*, value::Lazy, Error, ExceptionClass, IntoValue, Ruby, Value,
+};
+use sqlparser::ast::{
+    visit_expressions_mut, AssignmentTarget, Expr, FromTable, ObjectName, ObjectType, Query,
+    Statement, TableFactor, TableObject, TableWithJoins, Value as SqlValue, Visit, Visitor,
+};
 use sqlparser::{dialect::MySqlDialect, parser::Parser};
+use twox_hash::XxHash64;
 
 static PARSE_ERROR: Lazy<ExceptionClass> = Lazy::new(|ruby| {
     ruby.define_module("Finox")
@@ -8,16 +19,364 @@ static PARSE_ERROR: Lazy<ExceptionClass> = Lazy::new(|ruby| {
         .unwrap()
 });
 
-fn parse(ruby: &Ruby, sql: String) -> Result<Value, Error> {
+#[magnus::wrap(class = "Finox::Result", free_immediately, size)]
+struct ParseResult {
+    statements: Vec<Statement>,
+}
+
+impl ParseResult {
+    fn statements(ruby: &Ruby, rb_self: &Self) -> Result<Value, Error> {
+        let json = serde_json::to_value(&rb_self.statements)
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), e.to_string()))?;
+        json_to_ruby(ruby, &json)
+    }
+
+    fn tables(&self) -> Vec<String> {
+        dedup(self.statements.iter().flat_map(collect_tables).collect())
+    }
+
+    fn select_tables(&self) -> Vec<String> {
+        dedup(self.statements.iter().flat_map(collect_select_tables).collect())
+    }
+
+    fn dml_tables(&self) -> Vec<String> {
+        dedup(self.statements.iter().flat_map(collect_dml_tables).collect())
+    }
+
+    fn ddl_tables(&self) -> Vec<String> {
+        dedup(self.statements.iter().flat_map(collect_ddl_tables).collect())
+    }
+
+    fn columns(&self) -> Vec<String> {
+        collect_columns(&self.statements)
+    }
+
+    fn statement_types(&self) -> Vec<String> {
+        self.statements.iter().map(statement_type).collect()
+    }
+
+    fn normalize(&self) -> String {
+        self.statements
+            .iter()
+            .map(normalize_statement)
+            .collect::<Vec<_>>()
+            .join("; ")
+    }
+
+    fn fingerprint(&self) -> String {
+        fingerprint(&self.normalize())
+    }
+}
+
+fn fingerprint(sql: &str) -> String {
+    format!("{:016x}", XxHash64::oneshot(0, sql.as_bytes()))
+}
+
+fn json_to_ruby(ruby: &Ruby, value: &serde_json::Value) -> Result<Value, Error> {
+    use serde_json::Value as Json;
+
+    Ok(match value {
+        Json::Null => ruby.qnil().as_value(),
+        Json::Bool(boolean) => (*boolean).into_value_with(ruby),
+        Json::Number(number) => {
+            if let Some(int) = number.as_i64() {
+                int.into_value_with(ruby)
+            } else if let Some(int) = number.as_u64() {
+                int.into_value_with(ruby)
+            } else {
+                number.as_f64().into_value_with(ruby)
+            }
+        }
+        Json::String(string) => string.as_str().into_value_with(ruby),
+        Json::Array(items) => {
+            let array = ruby.ary_new_capa(items.len());
+            for item in items {
+                array.push(json_to_ruby(ruby, item)?)?;
+            }
+            array.as_value()
+        }
+        Json::Object(map) => {
+            let hash = ruby.hash_new();
+            for (key, item) in map {
+                hash.aset(key.as_str(), json_to_ruby(ruby, item)?)?;
+            }
+            hash.as_value()
+        }
+    })
+}
+
+fn normalize_statement(statement: &Statement) -> String {
+    let mut statement = statement.clone();
+    let _ = visit_expressions_mut(&mut statement, |expr| {
+        if let Expr::Value(value) = expr {
+            value.value = SqlValue::Placeholder("?".to_string());
+        }
+        ControlFlow::<()>::Continue(())
+    });
+    statement.to_string()
+}
+
+fn statement_type(statement: &Statement) -> String {
+    struct VariantName(String);
+
+    impl fmt::Write for VariantName {
+        fn write_str(&mut self, s: &str) -> fmt::Result {
+            for ch in s.chars() {
+                if ch.is_alphanumeric() || ch == '_' {
+                    self.0.push(ch);
+                } else {
+                    return Err(fmt::Error);
+                }
+            }
+            Ok(())
+        }
+    }
+
+    let mut name = VariantName(String::new());
+    let _ = write!(name, "{statement:?}");
+    name.0
+}
+
+fn table_info<T: Visit>(node: &T) -> TableCollector {
+    let mut collector = TableCollector::default();
+    let _ = node.visit(&mut collector);
+    collector
+}
+
+fn collect_tables<T: Visit>(node: &T) -> Vec<String> {
+    let info = table_info(node);
+    let mut names = info.relations;
+    names.extend(info.dml_targets);
+    names.extend(info.ddl_targets);
+    dedup(reject_cte_names(names, &info.cte_names))
+}
+
+fn collect_select_tables<T: Visit>(node: &T) -> Vec<String> {
+    let info = table_info(node);
+    let mut targets = info.dml_targets;
+    targets.extend(info.ddl_targets);
+
+    let mut names = Vec::new();
+    for name in info.relations {
+        match targets.iter().position(|target| *target == name) {
+            Some(position) => {
+                targets.remove(position);
+            }
+            None => names.push(name),
+        }
+    }
+    dedup(reject_cte_names(names, &info.cte_names))
+}
+
+fn collect_dml_tables<T: Visit>(node: &T) -> Vec<String> {
+    dedup(table_info(node).dml_targets)
+}
+
+fn collect_ddl_tables<T: Visit>(node: &T) -> Vec<String> {
+    dedup(table_info(node).ddl_targets)
+}
+
+fn reject_cte_names(names: Vec<String>, cte_names: &HashSet<String>) -> Vec<String> {
+    names
+        .into_iter()
+        .filter(|name| !cte_names.contains(name))
+        .collect()
+}
+
+fn collect_columns<T: Visit>(node: &T) -> Vec<String> {
+    let mut collector = ColumnCollector::default();
+    let _ = node.visit(&mut collector);
+    dedup(collector.columns)
+}
+
+fn dedup(names: Vec<String>) -> Vec<String> {
+    let mut result = Vec::new();
+    for name in names {
+        if !result.contains(&name) {
+            result.push(name);
+        }
+    }
+    result
+}
+
+fn object_name_to_string(name: &ObjectName) -> String {
+    name.0
+        .iter()
+        .map(|part| match part.as_ident() {
+            Some(ident) => ident.value.clone(),
+            None => part.to_string(),
+        })
+        .collect::<Vec<_>>()
+        .join(".")
+}
+
+fn table_with_joins_relation(table: &TableWithJoins) -> Option<String> {
+    match &table.relation {
+        TableFactor::Table { name, .. } => Some(object_name_to_string(name)),
+        _ => None,
+    }
+}
+
+#[derive(Default)]
+struct TableCollector {
+    relations: Vec<String>,
+    dml_targets: Vec<String>,
+    ddl_targets: Vec<String>,
+    cte_names: HashSet<String>,
+}
+
+impl Visitor for TableCollector {
+    type Break = ();
+
+    fn pre_visit_relation(&mut self, relation: &ObjectName) -> ControlFlow<()> {
+        self.relations.push(object_name_to_string(relation));
+        ControlFlow::Continue(())
+    }
+
+    fn pre_visit_query(&mut self, query: &Query) -> ControlFlow<()> {
+        if let Some(with) = &query.with {
+            for cte in &with.cte_tables {
+                self.cte_names.insert(cte.alias.name.value.clone());
+            }
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn pre_visit_statement(&mut self, statement: &Statement) -> ControlFlow<()> {
+        match statement {
+            Statement::Insert(insert) => {
+                if let TableObject::TableName(name) = &insert.table {
+                    self.dml_targets.push(object_name_to_string(name));
+                }
+            }
+            Statement::Update(update) => {
+                if let Some(name) = table_with_joins_relation(&update.table) {
+                    self.dml_targets.push(name);
+                }
+            }
+            Statement::Delete(delete) => {
+                if delete.tables.is_empty() {
+                    let (FromTable::WithFromKeyword(from) | FromTable::WithoutKeyword(from)) =
+                        &delete.from;
+                    for table in from {
+                        if let Some(name) = table_with_joins_relation(table) {
+                            self.dml_targets.push(name);
+                        }
+                    }
+                } else {
+                    for name in &delete.tables {
+                        self.dml_targets.push(object_name_to_string(name));
+                    }
+                }
+            }
+            Statement::CreateTable(create) => {
+                self.ddl_targets.push(object_name_to_string(&create.name));
+            }
+            Statement::AlterTable(alter) => {
+                self.ddl_targets.push(object_name_to_string(&alter.name));
+            }
+            Statement::CreateIndex(index) => {
+                self.ddl_targets
+                    .push(object_name_to_string(&index.table_name));
+            }
+            Statement::Drop {
+                object_type: ObjectType::Table,
+                names,
+                ..
+            } => {
+                for name in names {
+                    self.ddl_targets.push(object_name_to_string(name));
+                }
+            }
+            Statement::Truncate(truncate) => {
+                for target in &truncate.table_names {
+                    self.ddl_targets.push(object_name_to_string(&target.name));
+                }
+            }
+            Statement::RenameTable(renames) => {
+                for rename in renames {
+                    self.ddl_targets.push(object_name_to_string(&rename.old_name));
+                    self.ddl_targets.push(object_name_to_string(&rename.new_name));
+                }
+            }
+            _ => {}
+        }
+        ControlFlow::Continue(())
+    }
+}
+
+#[derive(Default)]
+struct ColumnCollector {
+    columns: Vec<String>,
+}
+
+impl Visitor for ColumnCollector {
+    type Break = ();
+
+    fn pre_visit_expr(&mut self, expr: &Expr) -> ControlFlow<()> {
+        match expr {
+            Expr::Identifier(ident) => self.columns.push(ident.value.clone()),
+            Expr::CompoundIdentifier(idents) => self.columns.push(
+                idents
+                    .iter()
+                    .map(|ident| ident.value.clone())
+                    .collect::<Vec<_>>()
+                    .join("."),
+            ),
+            _ => {}
+        }
+        ControlFlow::Continue(())
+    }
+
+    fn pre_visit_statement(&mut self, statement: &Statement) -> ControlFlow<()> {
+        match statement {
+            Statement::Insert(insert) => {
+                for column in &insert.columns {
+                    self.columns.push(object_name_to_string(column));
+                }
+            }
+            Statement::Update(update) => {
+                for assignment in &update.assignments {
+                    match &assignment.target {
+                        AssignmentTarget::ColumnName(name) => {
+                            self.columns.push(object_name_to_string(name));
+                        }
+                        AssignmentTarget::Tuple(names) => {
+                            for name in names {
+                                self.columns.push(object_name_to_string(name));
+                            }
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+        ControlFlow::Continue(())
+    }
+}
+
+fn parse(ruby: &Ruby, sql: String) -> Result<ParseResult, Error> {
     let statements = Parser::parse_sql(&MySqlDialect {}, &sql)
         .map_err(|e| Error::new(ruby.get_inner(&PARSE_ERROR), e.to_string()))?;
-    serde_magnus::serialize(ruby, &statements)
+    Ok(ParseResult { statements })
 }
 
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = ruby.define_module("Finox")?;
     Lazy::force(&PARSE_ERROR, ruby);
+
+    let result = module.define_class("Result", ruby.class_object())?;
+    result.define_method("statements", method!(ParseResult::statements, 0))?;
+    result.define_method("tables", method!(ParseResult::tables, 0))?;
+    result.define_method("select_tables", method!(ParseResult::select_tables, 0))?;
+    result.define_method("dml_tables", method!(ParseResult::dml_tables, 0))?;
+    result.define_method("ddl_tables", method!(ParseResult::ddl_tables, 0))?;
+    result.define_method("columns", method!(ParseResult::columns, 0))?;
+    result.define_method("statement_types", method!(ParseResult::statement_types, 0))?;
+    result.define_method("normalize", method!(ParseResult::normalize, 0))?;
+    result.define_method("fingerprint", method!(ParseResult::fingerprint, 0))?;
+
     module.define_singleton_method("parse", function!(parse, 1))?;
     Ok(())
 }

data/lib/finox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Finox
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: finox
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - kyuuri1791
@@ -30,11 +30,10 @@ extensions:
 - ext/finox/extconf.rb
 extra_rdoc_files: []
 files:
-- ".rspec"
-- ".rubocop.yml"
 - CHANGELOG.md
 - Cargo.lock
 - Cargo.toml
+- LICENSE-THIRD-PARTY.txt
 - LICENSE.txt
 - README.md
 - Rakefile

data/.rspec

@@ -1,3 +0,0 @@
---format documentation
---color
---require spec_helper

data/.rubocop.yml

@@ -1,8 +0,0 @@
-AllCops:
-  TargetRubyVersion: 3.1
-
-Style/StringLiterals:
-  EnforcedStyle: double_quotes
-
-Style/StringLiteralsInInterpolation:
-  EnforcedStyle: double_quotes