gmail_search_syntax 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b5e08a769d7b375473e7ca0e4afe134e03862ae3f31040c9bb22904ff482b33
-  data.tar.gz: 32624e727131b5bb0779f3f1271b6031e252fbdbdea776b445c598b06f343715
+  metadata.gz: d7c192efdd70d31b0747a356b1adab4c50160afb81e747d1088f2727f97c7262
+  data.tar.gz: 73f6401e73d2ecae59b6e6741c08786d2da3dffbbdcb6f21095c63bf40761f6d
 SHA512:
-  metadata.gz: d93ca6cb4e4d0bab18a9e3ff2620f669f9913cbafa3b389dd1bf3a828329df34be12f202856bba61e66d581dd8cf6c91a665889efeb8e5df3a50d5d60e33d131
-  data.tar.gz: c16da51e8f41ba6a9c001293df098c789b7dd5f3b494d3cfb0f60d3e16e485b8f5d185dd67c0cb1b0cade40d1079eca786c1614c7d7afc90585e97c5d0b8c4e1
+  metadata.gz: ce609cacd2a6276d9178276ee84e0ccffc80eb8840bd62e6402e2275817ba2052559f903a55648373e87a79527415d6be4df1d4052429c83e3faa9abc1fe2c1c
+  data.tar.gz: 7badbc6024a67752bfab201577ebeb900ba106ed9979110c04eed6c76d35e33e26a7b1147927a2fd9d329f6a7bc28b71d2487d0f26e87f23a9dd7f867fac7e09

data/.github/workflows/ci.yml

@@ -0,0 +1,24 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.4'
+        bundler-cache: true
+    
+    - name: Run tests
+      run: bundle exec rake
+

data/AGENTS.md

@@ -0,0 +1,104 @@
+# Agent Guidelines for gmail_search_syntax
+
+This document outlines the coding standards and workflow requirements for AI agents working on the gmail_search_syntax project.
+
+## Ruby File Standards
+
+### Frozen String Literal
+**ALWAYS** include `# frozen_string_literal: true` at the top of every Ruby file:
+
+```ruby
+# frozen_string_literal: true
+
+class MyClass
+  # ... implementation
+end
+```
+
+This directive should be the very first line of every `.rb` file to ensure string immutability and improve performance.
+
+## Documentation Standards
+
+### Markdown Files Location
+When writing step-by-step instructions, documentation, or process descriptions in Markdown format, place them in the `slop/` directory:
+
+```
+slop/
+├── ARCHITECTURE.md
+├── GMAIL_BEHAVIOR_COMPARISON.md
+├── GMAIL_COMPATIBILITY_COMPLETE.md
+└── IMPLEMENTATION_NOTES.md
+```
+
+This keeps the main project directory clean while preserving detailed documentation and implementation notes.
+
+## Code Formatting
+
+### StandardRB Integration
+After creating or modifying any Ruby file, **ALWAYS** run StandardRB to maintain consistent formatting:
+
+```bash
+standardrb --fix /path/to/file.rb
+```
+
+This ensures:
+- Consistent code style across the project
+- Automatic fixing of common formatting issues
+- Compliance with the project's Ruby style guide
+- Uniform indentation, spacing, and syntax
+
+### Workflow
+1. Create or modify a Ruby file
+2. Immediately run `standardrb --fix` on the file
+3. Verify the changes are acceptable
+4. Continue with development
+
+## Project Context
+
+This is a Ruby gem that parses Gmail's search syntax and converts it into an Abstract Syntax Tree (AST). The project includes:
+
+- **Core parsing**: Tokenizer, parser, and AST nodes
+- **SQL conversion**: SQLite and Postgres visitors for database queries
+- **Comprehensive testing**: Unit and integration tests
+- **Documentation**: Schema documentation and operator reference
+
+## Key Files
+
+- `lib/gmail_search_syntax.rb` - Main entry point
+- `lib/gmail_search_syntax/parser.rb` - Core parsing logic
+- `lib/gmail_search_syntax/sql_visitor.rb` - SQL generation
+- `test/` - Test suite
+- `SCHEMA.md` - Database schema documentation
+- `slop/` - Detailed implementation documentation
+
+## Best Practices
+
+1. **Test Coverage**: Ensure all new functionality has corresponding tests
+2. **Documentation**: Update relevant documentation when adding features
+3. **Backward Compatibility**: Maintain API compatibility when possible
+4. **Performance**: Consider performance implications of parsing changes
+5. **Gmail Compatibility**: Verify changes against Gmail's actual search behavior
+
+## Example Workflow
+
+```bash
+# 1. Create a new Ruby file
+echo '# frozen_string_literal: true' > lib/gmail_search_syntax/new_feature.rb
+
+# 2. Add implementation
+# ... write code ...
+
+# 3. Format with StandardRB
+standardrb --fix lib/gmail_search_syntax/new_feature.rb
+
+# 4. Create documentation if needed
+echo '# Implementation Notes' > slop/NEW_FEATURE_NOTES.md
+
+# 5. Add tests
+# ... write tests ...
+
+# 6. Run test suite
+bundle exec rake test
+```
+
+Remember: Consistency in code style and documentation organization is crucial for maintaining this project's quality and readability.

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -6,7 +6,8 @@ Based on the official Gmail search operators documentation:
 https://support.google.com/mail/answer/7190
 
 > [!TIP]
-> This gem was created for [Cora,](https://cora.computer/) your personal e-mail assistant. 
+> This gem was created for [Cora,](https://cora.computer/) 
+> your personal e-mail assistant. 
 > Send them some love for allowing me to share it.
 
 ## Installation
@@ -54,47 +55,6 @@ GmailSearchSyntax.parse!("")
 # => raises GmailSearchSyntax::EmptyQueryError
 ```
 
-### Converting to SQL
-
-The gem includes a SQLite visitor that can convert Gmail queries to SQL. Here's a complex example:
-
-```ruby
-require 'gmail_search_syntax'
-
-# A complex Gmail query with multiple operators
-query = '(from:manager OR from:boss) subject:"quarterly review" has:attachment -label:archived after:2024/01/01 larger:5M'
-
-ast = GmailSearchSyntax.parse!(query)
-visitor = GmailSearchSyntax::SQLiteVisitor.new(current_user_email: "user@example.com")
-visitor.visit(ast)
-
-sql, params = visitor.to_query.to_sql
-```
-
-This generates the following SQL:
-
-```sql
-SELECT DISTINCT m0.id 
-FROM messages AS m0 
-INNER JOIN message_addresses AS ma1 ON m0.id = ma1.message_id 
-INNER JOIN message_addresses AS ma3 ON m0.id = ma3.message_id 
-INNER JOIN message_labels AS ml ON m0.id = ml.message_id 
-INNER JOIN labels AS l ON ml.label_id = l.id 
-WHERE ((((ma1.address_type = ? OR ma1.address_type = ? OR ma1.address_type = ?) 
-         AND ma1.email_address = ?) 
-        OR ((ma3.address_type = ? OR ma3.address_type = ? OR ma3.address_type = ?) 
-            AND ma3.email_address = ?)) 
-       AND m0.subject LIKE ? 
-       AND m0.has_attachment = 1 
-       AND NOT l.name = ? 
-       AND m0.internal_date > ? 
-       AND m0.size_bytes > ?)
-```
-
-With parameters: `["from", "cc", "bcc", "manager", "from", "cc", "bcc", "boss", "%quarterly review%", "archived", "2024-01-01", 5242880]`
-
-A similar visitor is provided for PostgreSQL.
-
 ## Supported Operators
 
 Email routing: `from:`, `to:`, `cc:`, `bcc:`, `deliveredto:`  
@@ -113,6 +73,23 @@ Size: `size:`, `larger:`, `smaller:`
 
 There is also a converter from the operators to SQL queries against an embedded SQLite database. This is meant more as an example than a fully-featured store, but it shows what's possible.
 
+### Converting to SQL
+
+The gem includes a SQLite visitor and a Postgres visitor which converts the Gmail queries into corresponding SQL. See SCHEMA.md for more information.
+
+```ruby
+require 'gmail_search_syntax'
+
+# A complex Gmail query with multiple operators
+query = '(from:manager OR from:boss) subject:"quarterly review" has:attachment -label:archived after:2024/01/01 larger:5M'
+
+ast = GmailSearchSyntax.parse!(query)
+visitor = GmailSearchSyntax::SQLiteVisitor.new(current_user_email: "user@example.com")
+visitor.visit(ast)
+
+sql, params = visitor.to_query.to_sql
+```
+
 ## Testing
 
 ```bash

data/SCHEMA.md

@@ -1,15 +1,111 @@
-# Database Schema for Gmail Search
+This document describes the database schema designed to support Gmail search syntax queries. The schema is optimized for the search operators defined in `lib/GMAIL_SEARCH_OPERATORS.md`.
 
-## Overview
+```ruby
+require 'gmail_search_syntax'
 
-This document describes the SQLite database schema designed to support Gmail search syntax queries. The schema is optimized for the search operators defined in `lib/GMAIL_SEARCH_OPERATORS.md`.
+# A complex Gmail query with multiple operators
+query = '(from:manager OR from:boss) subject:"quarterly review" has:attachment -label:archived after:2024/01/01 larger:5M'
 
-## Core Tables
+ast = GmailSearchSyntax.parse!(query)
+visitor = GmailSearchSyntax::SQLiteVisitor.new(current_user_email: "user@example.com")
+visitor.visit(ast)
+
+sql, params = visitor.to_query.to_sql
+```
+
+generates the following SQL:
+
+```sql
+SELECT DISTINCT m0.id 
+FROM messages AS m0 
+INNER JOIN message_addresses AS ma1 ON m0.id = ma1.message_id 
+INNER JOIN message_addresses AS ma3 ON m0.id = ma3.message_id 
+INNER JOIN message_labels AS ml ON m0.id = ml.message_id 
+INNER JOIN labels AS l ON ml.label_id = l.id 
+WHERE ((((ma1.address_type = ? OR ma1.address_type = ? OR ma1.address_type = ?) 
+         AND ma1.email_address = ?) 
+        OR ((ma3.address_type = ? OR ma3.address_type = ? OR ma3.address_type = ?) 
+            AND ma3.email_address = ?)) 
+       AND m0.subject LIKE ? 
+       AND m0.has_attachment = 1 
+       AND NOT l.name = ? 
+       AND m0.internal_date > ? 
+       AND m0.size_bytes > ?)
+```
+
+and bound parameters:
+
+```
+[
+  "from", "cc", "bcc", "manager", "from", "cc", "bcc",
+  "boss", "%quarterly review%", "archived", "2024-01-01", 5242880
+]
+```
+
+## String Matching Requirements
+
+### Prefix/Suffix Matching
+Required for:
+- **Email addresses** (from:, to:, cc:, bcc:, deliveredto:)
+  - `from:marc@` → prefix match → `WHERE email_address LIKE 'marc@%'`
+  - `from:@example.com` → suffix match → `WHERE email_address LIKE '%@example.com'`
+  - `from:marc@example.com` → exact match → `WHERE email_address = 'marc@example.com'`
+
+- **Mailing lists** (list:)
+  - Same pattern as email addresses
+
+- **Filenames** (filename:)
+  - `filename:pdf` → extension match → `WHERE filename LIKE '%.pdf'`
+  - `filename:homework` → prefix match → `WHERE filename LIKE 'homework%'`
+
+### Exact Match Only
+- RFC822 message IDs
+- Boolean/enum fields (is:, has:, in:, category:, label:)
+
+## SQL Visitor Usage
+
+The library provides two SQL visitor implementations for different database backends: SQLite and Postgres. They are configured to use the schema described below. You convert the search AST nodes into a SQL query using the provided SQL visitors. If you have a different schema, use the visitor code as a template.
+
+
+```ruby
+ast = GmailSearchSyntax.parse!("from:amy@example.com newer_than:7d")
+visitor = GmailSearchSyntax::SQLiteVisitor.new(current_user_email: "me@example.com")
+visitor.visit(ast)
+
+sql, params = visitor.to_query.to_sql
+# sql: "SELECT DISTINCT m.id FROM messages m ... WHERE ... m.internal_date > datetime('now', ?)"
+# params: ["from", "cc", "bcc", "amy@example.com", "-7 days"]
+```
+
+The visitors implement:
+
+- **Parameterized queries**: All user input is bound via `?` placeholders
+- **Automatic table joins**: Joins required tables based on operators
+- **Nested conditions**: Properly handles AND/OR/NOT with parentheses
+- **Special operators**:
+  - `from:me` / `to:me` → uses `current_user_email`
+  - `in:anywhere` → no location filter
+  - `AROUND` → generates `(1 = 0)` no-op condition
+- **Date handling**:
+  - Converts dates from `YYYY/MM/DD` to `YYYY-MM-DD`
+  - Parses relative times (`1y`, `2d`, `3m`) to database-specific datetime functions
+- **Size parsing**: Converts `10M`, `1G` to bytes
+
+## Fuzzy Matching Limitations
+
+The current implementation does **not** support:
+- **AROUND operator** (proximity search) - generates no-op `(1 = 0)` condition
+- Full-text search with word distance calculations
+- Stemming or phonetic matching
+- Levenshtein distance / typo tolerance
+
+These features require additional implementation, potentially using SQLite FTS5 extensions.
 
-### messages
-Primary table storing email message metadata.
+## Core Tables
 
 ```sql
+-- messages
+-- Primary table storing email message metadata.
 CREATE TABLE messages (
     id TEXT PRIMARY KEY,
     rfc822_message_id TEXT,
@@ -55,12 +151,12 @@ CREATE TABLE messages (
     
     created_at DATETIME DEFAULT CURRENT_TIMESTAMP
 );
-```
+`
 
-### message_addresses
-Stores email addresses associated with messages (from, to, cc, bcc, delivered_to).
+-- message_addresses
+-- Stores email addresses associated with messages (from, to, cc, bcc, delivered_to).
+-- The `from:` and `to:` operators search across `from`, `cc`, and `bcc` address types per Gmail specification.
 
-```sql
 CREATE TABLE message_addresses (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     message_id TEXT NOT NULL,
@@ -70,26 +166,21 @@ CREATE TABLE message_addresses (
     
     FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE CASCADE
 );
-```
 
-**Note:** The `from:` and `to:` operators search across `from`, `cc`, and `bcc` address types per Gmail specification.
 
-### labels
-Label definitions with external string IDs.
+-- labels
+-- Label definitions with external string IDs.
 
-```sql
 CREATE TABLE labels (
     id TEXT PRIMARY KEY,
     name TEXT NOT NULL UNIQUE,
     is_system_label BOOLEAN DEFAULT 0,
     created_at DATETIME DEFAULT CURRENT_TIMESTAMP
 );
-```
 
-### message_labels
-Many-to-many relationship between messages and labels.
+-- message_labels
+-- Many-to-many relationship between messages and labels.
 
-```sql
 CREATE TABLE message_labels (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     message_id TEXT NOT NULL,
@@ -100,11 +191,9 @@ CREATE TABLE message_labels (
     UNIQUE(message_id, label_id)
 );
 ```
+-- attachments
+-- File attachments associated with messages.
 
-### attachments
-File attachments associated with messages.
-
-```sql
 CREATE TABLE attachments (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     message_id TEXT NOT NULL,
@@ -116,108 +205,3 @@ CREATE TABLE attachments (
 );
 ```
 
-## String Matching Requirements
-
-### Prefix/Suffix Matching
-Required for:
-- **Email addresses** (from:, to:, cc:, bcc:, deliveredto:)
-  - `from:marc@` → prefix match → `WHERE email_address LIKE 'marc@%'`
-  - `from:@example.com` → suffix match → `WHERE email_address LIKE '%@example.com'`
-  - `from:marc@example.com` → exact match → `WHERE email_address = 'marc@example.com'`
-
-- **Mailing lists** (list:)
-  - Same pattern as email addresses
-
-- **Filenames** (filename:)
-  - `filename:pdf` → extension match → `WHERE filename LIKE '%.pdf'`
-  - `filename:homework` → prefix match → `WHERE filename LIKE 'homework%'`
-
-### Exact Match Only
-- RFC822 message IDs
-- Boolean/enum fields (is:, has:, in:, category:, label:)
-
-## SQL Visitor Usage
-
-The library provides two SQL visitor implementations for different database backends:
-
-### SQLiteVisitor
-
-Converts parsed AST into SQLite-compatible SQL queries:
-
-```ruby
-ast = GmailSearchSyntax.parse!("from:amy@example.com newer_than:7d")
-visitor = GmailSearchSyntax::SQLiteVisitor.new(current_user_email: "me@example.com")
-visitor.visit(ast)
-
-sql, params = visitor.to_query.to_sql
-# sql: "SELECT DISTINCT m.id FROM messages m ... WHERE ... m.internal_date > datetime('now', ?)"
-# params: ["from", "cc", "bcc", "amy@example.com", "-7 days"]
-```
-
-### PostgresVisitor
-
-Converts parsed AST into PostgreSQL-compatible SQL queries:
-
-```ruby
-ast = GmailSearchSyntax.parse!("from:amy@example.com newer_than:7d")
-visitor = GmailSearchSyntax::PostgresVisitor.new(current_user_email: "me@example.com")
-visitor.visit(ast)
-
-sql, params = visitor.to_query.to_sql
-# sql: "SELECT DISTINCT m.id FROM messages m ... WHERE ... m.internal_date > (NOW() - ?::interval)"
-# params: ["from", "cc", "bcc", "amy@example.com", "7 days"]
-```
-
-**Note**: `SqlVisitor` is an alias for `SQLiteVisitor` for backward compatibility.
-
-### Database-Specific Differences
-
-The main difference between the visitors is in relative date handling:
-
-| Feature | SQLite | PostgreSQL |
-|---------|--------|------------|
-| `older_than:7d` | `datetime('now', '-7 days')` | `NOW() - '7 days'::interval` |
-| `newer_than:3m` | `datetime('now', '-3 months')` | `NOW() - '3 months'::interval` |
-| Parameter format | `"-7 days"` (negative) | `"7 days"` (positive with cast) |
-
-All other query generation is identical between the two visitors.
-
-### Features
-
-- **Parameterized queries**: All user input is bound via `?` placeholders
-- **Automatic table joins**: Joins required tables based on operators
-- **Nested conditions**: Properly handles AND/OR/NOT with parentheses
-- **Special operators**:
-  - `from:me` / `to:me` → uses `current_user_email`
-  - `in:anywhere` → no location filter
-  - `AROUND` → generates `(1 = 0)` no-op condition
-- **Date handling**:
-  - Converts dates from `YYYY/MM/DD` to `YYYY-MM-DD`
-  - Parses relative times (`1y`, `2d`, `3m`) to database-specific datetime functions
-- **Size parsing**: Converts `10M`, `1G` to bytes
-
-### Query Object
-
-The `Query` class accumulates SQL components:
-
-```ruby
-query = visitor.to_query
-
-query.conditions  # Array of WHERE conditions
-query.joins       # Hash of JOIN clauses
-query.params      # Array of bound parameters
-
-sql, params = query.to_sql
-# Returns: [sql_string, parameters_array]
-```
-
-## Fuzzy Matching Limitations
-
-The current implementation does **not** support:
-- **AROUND operator** (proximity search) - generates no-op `(1 = 0)` condition
-- Full-text search with word distance calculations
-- Stemming or phonetic matching
-- Levenshtein distance / typo tolerance
-
-These features require additional implementation, potentially using SQLite FTS5 extensions.
-

data/examples/demo.rb

@@ -1,8 +1,5 @@
 require_relative "../lib/gmail_search_syntax"
-
-puts "Gmail Search Syntax Parser - Demo"
-puts "=" * 50
-puts
+require "pp"
 
 queries = [
   "from:amy@example.com",
@@ -23,6 +20,5 @@ queries = [
 queries.each do |query|
   puts "Query: #{query}"
   ast = GmailSearchSyntax.parse!(query)
-  puts "AST:   #{ast.inspect}"
-  puts
+  pp ast
 end

data/gmail_search_syntax.gemspec

@@ -0,0 +1,23 @@
+require_relative "lib/gmail_search_syntax/version"
+
+Gem::Specification.new do |s|
+  s.name = "gmail_search_syntax"
+  s.version = GmailSearchSyntax::VERSION
+  s.summary = "Gmail search syntax parser"
+  s.authors = ["me@julik.nl"]
+  s.license = "MIT"
+  s.homepage = "https://github.com/julik/gmail_search_syntax"
+  s.required_ruby_version = ">= 3.0"
+
+  s.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      File.basename(f).start_with?(".")
+    end
+  end
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "minitest", "~> 5.0"
+  s.add_development_dependency "rake", "~> 13.0"
+  s.add_development_dependency "sqlite3", "< 1.6"
+  s.add_development_dependency "standard", "~> 1.0"
+end

data/lib/gmail_search_syntax/parser.rb

@@ -195,47 +195,17 @@ module GmailSearchSyntax
       when :lbrace
         parse_braces
       when :quoted_string
-        # Quoted strings are consumed as-is, no bareword collection
         value = current_token.value
         advance
         value
       when :word, :email, :number, :date, :relative_time
-        # Collect the initial value and any following barewords
-        # until we hit an operator, special token, or grouping
-        values = []
-        types = []
-
-        # Collect barewords
-        while !eof? && is_bareword_token?
-          # Check if this word is actually an operator (word followed by colon)
-          if current_token.type == :word && peek_token&.type == :colon
-            break
-          end
-
-          values << current_token.value
-          types << current_token.type
-          advance
-        end
-
-        # If we only collected one value and it's a number, preserve its type
-        if values.length == 1 && types[0] == :number
-          values[0]
-        else
-          # Multiple values or non-number: join as string
-          values.map(&:to_s).join(" ")
-        end
-      end
-    end
-
-    def is_bareword_token?
-      return false if eof?
-
-      # Barewords are simple value tokens, not operators or special syntax
-      case current_token.type
-      when :word, :email, :number, :date, :relative_time
-        true
-      else
-        false
+        # Take only a single token as the operator value.
+        # Multi-word values must be explicitly quoted: from:"john smith"
+        # This matches Gmail's actual search behavior where bare words
+        # after an operator are treated as separate search terms.
+        value = current_token.value
+        advance
+        value.is_a?(Integer) ? value : value.to_s
       end
     end
   end

data/lib/gmail_search_syntax/version.rb

@@ -1,3 +1,3 @@
 module GmailSearchSyntax
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/slop/GREEDY_VS_NON_GREEDY_TOKENIZATION.md

@@ -0,0 +1,84 @@
+# Greedy vs Non-Greedy Operator Value Tokenization
+
+## Summary
+
+This document explains the fix in `bugfix-tokens` that changes how operator values are parsed from **greedy** (consuming multiple barewords) to **non-greedy** (single token only), matching Gmail's actual search behavior.
+
+## The Problem
+
+The previous implementation used greedy tokenization for operator values. When parsing `label:Cora/Google Drive`, the parser would consume all subsequent barewords (`Cora/Google`, `Drive`) into the operator's value until hitting another operator or special token.
+
+**Previous behavior:**
+```
+label:Cora/Google Drive label:Notes
+→ Operator(label, "Cora/Google Drive"), Operator(label, "Notes")
+```
+
+**Gmail's actual behavior:**
+```
+label:Cora/Google Drive label:Notes
+→ Operator(label, "Cora/Google"), StringToken("Drive"), Operator(label, "Notes")
+```
+
+## Gmail's Actual Behavior
+
+In Gmail search, barewords after an operator are treated as **separate search terms**, not as part of the operator's value. To include multiple words in an operator value, you must explicitly quote them:
+
+| Input | Gmail Interpretation |
+|-------|---------------------|
+| `subject:urgent meeting` | Subject contains "urgent" AND body contains "meeting" |
+| `subject:"urgent meeting"` | Subject contains "urgent meeting" |
+| `in:anywhere movie` | Search "movie" in all mail locations |
+| `label:test one two` | Label is "test" AND body contains "one" AND "two" |
+
+## The Fix
+
+Changed `parse_operator_value` in `lib/gmail_search_syntax/parser.rb` to only consume a single token for bareword values (`:word`, `:email`, `:number`, `:date`, `:relative_time`).
+
+### Before (greedy)
+
+```ruby
+when :word, :email, :number, :date, :relative_time
+  values = []
+  types = []
+
+  # Collect barewords until operator or special token
+  while !eof? && is_bareword_token?
+    if current_token.type == :word && peek_token&.type == :colon
+      break
+    end
+    values << current_token.value
+    types << current_token.type
+    advance
+  end
+
+  # Join multiple values as string
+  values.map(&:to_s).join(" ")
+```
+
+### After (non-greedy)
+
+```ruby
+when :word, :email, :number, :date, :relative_time
+  # Take only a single token as the operator value.
+  # Multi-word values must be explicitly quoted: from:"john smith"
+  value = current_token.value
+  advance
+  value.is_a?(Integer) ? value : value.to_s
+```
+
+## Test Changes
+
+Updated tests to reflect the corrected behavior:
+
+| Test | Previous Expected | Now Expected |
+|------|-------------------|--------------|
+| `in:anywhere movie` | `Operator("in", "anywhere movie")` | `Operator("in", "anywhere")`, `StringToken("movie")` |
+| `subject:urgent meeting important` | `Operator("subject", "urgent meeting important")` | `Operator("subject", "urgent")`, `StringToken("meeting")`, `StringToken("important")` |
+| `label:test one two three label:another` | 2 operands | 5 operands |
+
+## Implications
+
+1. **Breaking change** for consumers relying on greedy behavior
+2. Users must now quote multi-word operator values explicitly
+3. More accurate translation to SQL/other query languages since the semantics now match Gmail

data/test/gmail_search_syntax_test.rb

@@ -200,10 +200,13 @@ class GmailSearchSyntaxTest < Minitest::Test
   end
 
   def test_in_anywhere
-    # With Gmail-compatible bareword consumption, "movie" gets consumed into operator value
-    # To search for "movie" as text, use: in:anywhere "movie" or use a different operator after
+    # Gmail treats barewords after operator as separate search terms
+    # in:anywhere movie → search for "movie" in all mail locations
     ast = GmailSearchSyntax.parse!("in:anywhere movie")
-    assert_operator({name: "in", value: "anywhere movie"}, ast)
+    assert_instance_of And, ast
+    assert_equal 2, ast.operands.length
+    assert_operator({name: "in", value: "anywhere"}, ast.operands[0])
+    assert_string_token({value: "movie"}, ast.operands[1])
   end
 
   def test_is_starred
@@ -660,92 +663,103 @@ class GmailSearchSyntaxTest < Minitest::Test
     assert_equal 'project\\plan', ast.operands[1].value
   end
 
-  # Gmail behavior: barewords after operator values get consumed into the operator value
-  # until the next operator is encountered.
-  # We now implement this Gmail-compatible behavior.
+  # Gmail behavior: barewords after operator values are treated as separate search terms.
+  # Multi-word operator values must be explicitly quoted: label:"Cora/Google Drive"
 
   def test_label_with_space_separated_value_gmail_behavior
-    # Gmail parses this as: label:"Cora/Google Drive", label:"Notes"
-    # We now match this behavior
+    # Gmail treats barewords as separate search terms
+    # To search for label "Cora/Google Drive", you must quote it: label:"Cora/Google Drive"
     ast = GmailSearchSyntax.parse!("label:Cora/Google Drive label:Notes")
     assert_instance_of And, ast
-    assert_equal 2, ast.operands.length
+    assert_equal 3, ast.operands.length
 
-    # Gmail-compatible: barewords consumed into operator value
+    # First operator takes only the first token
     assert_instance_of Operator, ast.operands[0]
     assert_equal "label", ast.operands[0].name
-    assert_equal "Cora/Google Drive", ast.operands[0].value
+    assert_equal "Cora/Google", ast.operands[0].value
+
+    # "Drive" becomes a separate search term
+    assert_instance_of StringToken, ast.operands[1]
+    assert_equal "Drive", ast.operands[1].value
 
     # Second operator parsed correctly
-    assert_instance_of Operator, ast.operands[1]
-    assert_equal "label", ast.operands[1].name
-    assert_equal "Notes", ast.operands[1].value
+    assert_instance_of Operator, ast.operands[2]
+    assert_equal "label", ast.operands[2].name
+    assert_equal "Notes", ast.operands[2].value
   end
 
   def test_subject_with_barewords_gmail_behavior
-    # Gmail parses: subject:"urgent meeting important"
-    # We now match this behavior
+    # Gmail treats barewords as separate search terms
+    # subject:urgent meeting important → subject contains "urgent" AND body contains "meeting" AND "important"
     ast = GmailSearchSyntax.parse!("subject:urgent meeting important")
-    assert_instance_of Operator, ast
+    assert_instance_of And, ast
+    assert_equal 3, ast.operands.length
 
-    assert_equal "subject", ast.name
-    assert_equal "urgent meeting important", ast.value
+    assert_operator({name: "subject", value: "urgent"}, ast.operands[0])
+    assert_string_token({value: "meeting"}, ast.operands[1])
+    assert_string_token({value: "important"}, ast.operands[2])
   end
 
   def test_multiple_barewords_between_operators_gmail_behavior
-    # Gmail parses: label:"test one two three", label:"another"
-    # We now match this behavior
+    # Gmail treats each bareword as a separate search term
+    # label:test one two three label:another → 5 terms
     ast = GmailSearchSyntax.parse!("label:test one two three label:another")
     assert_instance_of And, ast
-    assert_equal 2, ast.operands.length
+    assert_equal 5, ast.operands.length
 
-    assert_instance_of Operator, ast.operands[0]
-    assert_equal "label", ast.operands[0].name
-    assert_equal "test one two three", ast.operands[0].value
-
-    assert_instance_of Operator, ast.operands[1]
-    assert_equal "label", ast.operands[1].name
-    assert_equal "another", ast.operands[1].value
+    assert_operator({name: "label", value: "test"}, ast.operands[0])
+    assert_string_token({value: "one"}, ast.operands[1])
+    assert_string_token({value: "two"}, ast.operands[2])
+    assert_string_token({value: "three"}, ast.operands[3])
+    assert_operator({name: "label", value: "another"}, ast.operands[4])
   end
 
   def test_barewords_stop_at_special_operators
-    # Bareword collection should stop at OR, AND, AROUND
+    # Barewords are separate terms, OR separates two groups
     ast = GmailSearchSyntax.parse!("subject:urgent meeting OR subject:important call")
     assert_instance_of Or, ast
     assert_equal 2, ast.operands.length
 
-    assert_instance_of Operator, ast.operands[0]
-    assert_equal "subject", ast.operands[0].name
-    assert_equal "urgent meeting", ast.operands[0].value
+    # Left side: subject:urgent AND meeting (implicit AND)
+    assert_instance_of And, ast.operands[0]
+    assert_equal 2, ast.operands[0].operands.length
+    assert_operator({name: "subject", value: "urgent"}, ast.operands[0].operands[0])
+    assert_string_token({value: "meeting"}, ast.operands[0].operands[1])
 
-    assert_instance_of Operator, ast.operands[1]
-    assert_equal "subject", ast.operands[1].name
-    assert_equal "important call", ast.operands[1].value
+    # Right side: subject:important AND call (implicit AND)
+    assert_instance_of And, ast.operands[1]
+    assert_equal 2, ast.operands[1].operands.length
+    assert_operator({name: "subject", value: "important"}, ast.operands[1].operands[0])
+    assert_string_token({value: "call"}, ast.operands[1].operands[1])
   end
 
   def test_barewords_with_mixed_tokens
-    # Numbers, dates, emails should all be collected as barewords
+    # Numbers, dates, emails are all separate search terms
     ast = GmailSearchSyntax.parse!("subject:meeting 2024 Q1 review")
-    assert_instance_of Operator, ast
-    assert_equal "subject", ast.name
-    assert_equal "meeting 2024 Q1 review", ast.value
+    assert_instance_of And, ast
+    assert_equal 4, ast.operands.length
+
+    assert_operator({name: "subject", value: "meeting"}, ast.operands[0])
+    assert_string_token({value: 2024}, ast.operands[1])
+    assert_string_token({value: "Q1"}, ast.operands[2])
+    assert_string_token({value: "review"}, ast.operands[3])
   end
 
   def test_specific_gmail_example_cora_google_drive
-    # The specific example from the user: label:Cora/Google Drive label:Notes
-    # This should parse as two separate label operators with multi-word values
+    # label:Cora/Google Drive label:Notes
+    # "Drive" is a separate search term - to include it in the label, quote it:
+    # label:"Cora/Google Drive" label:Notes
     ast = GmailSearchSyntax.parse!("label:Cora/Google Drive label:Notes")
     assert_instance_of And, ast
-    assert_equal 2, ast.operands.length
+    assert_equal 3, ast.operands.length
 
-    # First operator: label with "Cora/Google Drive"
-    assert_instance_of Operator, ast.operands[0]
-    assert_equal "label", ast.operands[0].name
-    assert_equal "Cora/Google Drive", ast.operands[0].value
+    # First operator: label with "Cora/Google" only
+    assert_operator({name: "label", value: "Cora/Google"}, ast.operands[0])
+
+    # "Drive" becomes a separate search term
+    assert_string_token({value: "Drive"}, ast.operands[1])
 
     # Second operator: label with "Notes"
-    assert_instance_of Operator, ast.operands[1]
-    assert_equal "label", ast.operands[1].name
-    assert_equal "Notes", ast.operands[1].value
+    assert_operator({name: "label", value: "Notes"}, ast.operands[2])
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gmail_search_syntax
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - me@julik.nl
@@ -69,10 +69,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ARCHITECTURE.md
-- GMAIL_BEHAVIOR_COMPARISON.md
-- GMAIL_COMPATIBILITY_COMPLETE.md
-- IMPLEMENTATION_NOTES.md
+- ".github/workflows/ci.yml"
+- AGENTS.md
+- Gemfile
 - README.md
 - Rakefile
 - SCHEMA.md
@@ -84,6 +83,7 @@ files:
 - examples/postgres_vs_sqlite.rb
 - examples/sql_query.rb
 - examples/text_vs_substring_demo.rb
+- gmail_search_syntax.gemspec
 - lib/GMAIL_SEARCH_OPERATORS.md
 - lib/gmail_search_syntax.rb
 - lib/gmail_search_syntax/ast.rb
@@ -91,6 +91,11 @@ files:
 - lib/gmail_search_syntax/sql_visitor.rb
 - lib/gmail_search_syntax/tokenizer.rb
 - lib/gmail_search_syntax/version.rb
+- slop/ARCHITECTURE.md
+- slop/GMAIL_BEHAVIOR_COMPARISON.md
+- slop/GMAIL_COMPATIBILITY_COMPLETE.md
+- slop/GREEDY_VS_NON_GREEDY_TOKENIZATION.md
+- slop/IMPLEMENTATION_NOTES.md
 - test/gmail_search_syntax_test.rb
 - test/integration_test.rb
 - test/postgres_visitor_test.rb