gmail_search_syntax 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7c192efdd70d31b0747a356b1adab4c50160afb81e747d1088f2727f97c7262
-  data.tar.gz: 73f6401e73d2ecae59b6e6741c08786d2da3dffbbdcb6f21095c63bf40761f6d
+  metadata.gz: b9a9c5131ce88b747e30a23baa0c32943ccc820097b288c9cec7b8275ec5bdd8
+  data.tar.gz: 4f52b8ad66ca0b8894ee3105ca0f9d9cb527dcf4910284ff9f53811f53ebdb81
 SHA512:
-  metadata.gz: ce609cacd2a6276d9178276ee84e0ccffc80eb8840bd62e6402e2275817ba2052559f903a55648373e87a79527415d6be4df1d4052429c83e3faa9abc1fe2c1c
-  data.tar.gz: 7badbc6024a67752bfab201577ebeb900ba106ed9979110c04eed6c76d35e33e26a7b1147927a2fd9d329f6a7bc28b71d2487d0f26e87f23a9dd7f867fac7e09
+  metadata.gz: 9ad29d2d10821dc8a1bc1da6cb778f59979aebeaa92daf298939a67ec0b55dd22b7d29911dd64bf43c4564a193120d8bef3efae29c57cc822b35c709cf5b51bf
+  data.tar.gz: 1c932b207aa8bceafb7edcf439cae91e144975e3af62a6ccc78eca2f946f0b101f090af7ec58f9ca687426eeaab4e0476183cf1fe5f3a332abfff0dbb6931e29

data/examples/escaped_quotes_demo.rb

@@ -8,7 +8,7 @@ puts "=" * 80
 puts
 
 # Example 1: Escaped quotes in a substring search
-puts "1. Substring with escaped quotes"
+puts "1. ExactWord with escaped quotes"
 puts "-" * 40
 query1 = '"She said \\"hello\\" to me"'
 puts "Input: #{query1}"
@@ -136,11 +136,11 @@ puts "Summary"
 puts "=" * 80
 puts "Escape sequences work in BOTH quoted and unquoted tokens:"
 puts
-puts "Quoted strings (Substring nodes):"
+puts "Quoted strings (ExactWord nodes):"
 puts "  - Use substring matching (LIKE %value%)"
 puts "  - \"She said \\\"hello\\\"\" → 'She said \"hello\"'"
 puts
-puts "Unquoted tokens (StringToken nodes):"
+puts "Unquoted tokens (LooseWord nodes):"
 puts "  - Use word boundary matching (= or LIKE with boundaries)"
 puts "  - meeting\\\"room → 'meeting\"room'"
 puts "  - path\\\\to\\\\file → 'path\\to\\file'"

data/examples/text_vs_substring_demo.rb

@@ -3,11 +3,11 @@
 require_relative "../lib/gmail_search_syntax"
 
 puts "=" * 80
-puts "StringToken vs Substring Demo"
+puts "LooseWord vs ExactWord Demo"
 puts "=" * 80
 puts
 
-# Example 1: Unquoted text (StringToken node) - word boundary matching
+# Example 1: Unquoted text (LooseWord node) - word boundary matching
 puts "1. Unquoted text: meeting"
 puts "-" * 40
 query1 = "meeting"
@@ -27,7 +27,7 @@ puts "  - Will match: 'meeting tomorrow', 'the meeting', 'just a meeting here'"
 puts "  - Will NOT match: 'meetings', 'premeeting', 'meetingroom'"
 puts
 
-# Example 2: Quoted text (Substring node) - substring matching
+# Example 2: Quoted text (ExactWord node) - substring matching
 puts "2. Quoted text: \"meeting\""
 puts "-" * 40
 query2 = '"meeting"'
@@ -88,6 +88,6 @@ puts
 puts "=" * 80
 puts "Summary"
 puts "=" * 80
-puts "StringToken node (unquoted):  Word boundary matching - finds complete words"
-puts "Substring node (quoted):      Substring matching - finds partial matches"
+puts "LooseWord node (unquoted):  Word boundary matching - finds complete words"
+puts "ExactWord node (quoted):      ExactWord matching - finds partial matches"
 puts "=" * 80

data/lib/gmail_search_syntax/ast.rb

@@ -8,6 +8,10 @@ module GmailSearchSyntax
       def attributes
         instance_variables.map { |var| instance_variable_get(var) }
       end
+
+      def short_class_name
+        self.class.name.split("::").last
+      end
     end
 
     class Operator < Node
@@ -19,11 +23,11 @@ module GmailSearchSyntax
       end
 
       def inspect
-        "#<Operator #{@name}: #{@value.inspect}>"
+        "#<#{short_class_name} #{@name}: #{@value.inspect}>"
       end
     end
 
-    class StringToken < Node
+    class LooseWord < Node
       attr_reader :value
 
       def initialize(value)
@@ -31,11 +35,11 @@ module GmailSearchSyntax
       end
 
       def inspect
-        "#<StringToken #{@value.inspect}>"
+        "#<#{short_class_name} #{@value.inspect}>"
       end
     end
 
-    class Substring < Node
+    class ExactWord < Node
       attr_reader :value
 
       def initialize(value)
@@ -43,7 +47,7 @@ module GmailSearchSyntax
       end
 
       def inspect
-        "#<Substring #{@value.inspect}>"
+        "#<#{short_class_name} #{@value.inspect}>"
       end
     end
 
@@ -55,7 +59,7 @@ module GmailSearchSyntax
       end
 
       def inspect
-        "#<And #{@operands.map(&:inspect).join(" AND ")}>"
+        "#<#{short_class_name} #{@operands.map(&:inspect).join(" AND ")}>"
       end
     end
 
@@ -67,7 +71,7 @@ module GmailSearchSyntax
       end
 
       def inspect
-        "#<Or #{@operands.map(&:inspect).join(" OR ")}>"
+        "#<#{short_class_name} #{@operands.map(&:inspect).join(" OR ")}>"
       end
     end
 
@@ -79,7 +83,7 @@ module GmailSearchSyntax
       end
 
       def inspect
-        "#<Not #{@child.inspect}>"
+        "#<#{short_class_name} #{@child.inspect}>"
       end
     end
 
@@ -91,7 +95,7 @@ module GmailSearchSyntax
       end
 
       def inspect
-        "#<Group #{@children.inspect}>"
+        "#<#{short_class_name} #{@children.inspect}>"
       end
     end
 
@@ -105,7 +109,7 @@ module GmailSearchSyntax
       end
 
       def inspect
-        "#<Around #{@left.inspect} AROUND #{@distance} #{@right.inspect}>"
+        "#<#{short_class_name} #{@left.inspect} AROUND #{@distance} #{@right.inspect}>"
       end
     end
   end

data/lib/gmail_search_syntax/parser.rb

@@ -23,7 +23,11 @@ module GmailSearchSyntax
         raise GmailSearchSyntax::EmptyQueryError, "Query cannot be empty"
       end
 
-      children.first
+      # A token the grammar has no use for (a stray parenthesis, a colon
+      # after a space) ends an expression; whatever follows is still part of
+      # the query, so the pieces are joined the way adjacent terms are.
+      return children.first if children.length == 1
+      AST::And.new(children.flat_map { |child| child.is_a?(AST::And) ? child.operands : [child] })
     end
 
     private
@@ -129,11 +133,11 @@ module GmailSearchSyntax
       when :quoted_string
         value = current_token.value
         advance
-        AST::Substring.new(value)
+        AST::ExactWord.new(value)
       when :email, :number, :date, :relative_time
         value = current_token.value
         advance
-        AST::StringToken.new(value)
+        AST::LooseWord.new(value)
       else
         advance
         nil
@@ -171,7 +175,8 @@ module GmailSearchSyntax
     end
 
     def parse_operator_or_text
-      word = current_token.value
+      word_token = current_token
+      word = word_token.value
 
       if OPERATORS.include?(word.downcase) && peek_token&.type == :colon
         operator_name = word.downcase
@@ -183,7 +188,28 @@ module GmailSearchSyntax
       end
 
       advance
-      AST::StringToken.new(word)
+      AST::LooseWord.new(glued_text(word_token))
+    end
+
+    # A word that is not an operator keeps whatever is glued to it through
+    # colons, so "http://example.com", "10:30" and "foo:bar" are searched as
+    # typed, the way Gmail does. A colon followed by whitespace is a
+    # separator: "foo: bar" is "foo" and "bar".
+    def glued_text(word_token)
+      text = word_token.value.to_s
+      cursor = word_token.position + text.length
+      while current_token&.type == :colon && current_token.position == cursor
+        advance
+        cursor += 1
+        text += ":"
+        piece_token = current_token
+        break unless piece_token && %i[word email number date relative_time].include?(piece_token.type) && piece_token.position == cursor
+        piece = piece_token.value.to_s
+        text += piece
+        cursor += piece.length
+        advance
+      end
+      text.sub(/:+\z/, "")
     end
 
     def parse_operator_value

data/lib/gmail_search_syntax/sql_visitor.rb

@@ -46,10 +46,10 @@ module GmailSearchSyntax
       case node
       when AST::Operator
         visit_operator(node)
-      when AST::StringToken
-        visit_string_token(node)
-      when AST::Substring
-        visit_substring(node)
+      when AST::LooseWord
+        visit_loose_word(node)
+      when AST::ExactWord
+        visit_exact_word(node)
       when AST::And
         visit_and(node)
       when AST::Or
@@ -327,7 +327,7 @@ module GmailSearchSyntax
       @query.add_condition("m0.rfc822_message_id = ?")
     end
 
-    def visit_string_token(node)
+    def visit_loose_word(node)
       # Word boundary matching - the value should appear as a complete word/token
       # We use LIKE with word boundaries: spaces, start/end of string
       value = node.value
@@ -342,8 +342,8 @@ module GmailSearchSyntax
       @query.add_param("% #{value} %")
     end
 
-    def visit_substring(node)
-      # Substring matching - the value can appear anywhere in the text
+    def visit_exact_word(node)
+      # ExactWord matching - the value can appear anywhere in the text
       @query.add_param("%#{node.value}%")
       @query.add_param("%#{node.value}%")
       @query.add_condition("(m0.subject LIKE ? OR m0.body LIKE ?)")

data/lib/gmail_search_syntax/tokenizer.rb

@@ -38,6 +38,9 @@ module GmailSearchSyntax
 
         break if @position >= @input.length
 
+        # Every token made in this pass starts here; a value read right
+        # after an operator's colon records its own start below.
+        @token_start = @position
         char = current_char
 
         case char
@@ -55,9 +58,16 @@ module GmailSearchSyntax
           advance
         when "-"
           next_char = peek_char
-          if next_char && next_char !~ /\s/
+          prev_char = (@position > 0) ? @input[@position - 1] : nil
+          # Negation requires: non-whitespace follows AND (start of input OR whitespace precedes)
+          # Gmail behavior: "Coxlee-Gammage" → Coxlee AND Gammage (hyphen is word separator)
+          #                 "Coxlee -Gammage" → Coxlee AND NOT Gammage (space+hyphen = negation)
+          if next_char && next_char !~ /\s/ && (prev_char.nil? || prev_char =~ /\s/)
             add_token(:minus, char)
             advance
+          elsif prev_char && prev_char !~ /\s/
+            # Embedded hyphen (preceded by non-whitespace) - skip it as word separator
+            advance
           else
             read_word
           end
@@ -69,11 +79,18 @@ module GmailSearchSyntax
         when ":"
           add_token(:colon, char)
           advance
+          # After a colon following an operator word, read the value including hyphens
+          # Gmail: label:foo-bar → operator "label" with value "foo-bar"
+          if in_operator_context? && @position < @input.length && current_char !~ /[\s(){}]/
+            @token_start = @position
+            read_operator_value
+          end
         else
           read_word
         end
       end
 
+      @token_start = @position
       add_token(:eof, nil)
       @tokens
     end
@@ -98,8 +115,72 @@ module GmailSearchSyntax
       end
     end
 
+    # Tokens carry the offset they start at, so the parser can tell "foo:bar"
+    # from "foo: bar".
     def add_token(type, value)
-      @tokens << Token.new(type, value, @position)
+      @tokens << Token.new(type, value, @token_start || @position)
+    end
+
+    # Check if we're in operator context (last non-colon token is an operator word)
+    def in_operator_context?
+      # Find the last token before the colon we just added
+      return false if @tokens.size < 2
+      prev_token = @tokens[-2]
+      prev_token.type == :word && OPERATORS.include?(prev_token.value.downcase)
+    end
+
+    # Read an operator value, which can include hyphens
+    # Gmail treats label:foo-bar as operator "label" with value "foo-bar"
+    def read_operator_value
+      # Handle quoted strings normally
+      if current_char == '"'
+        read_quoted_string
+        return
+      end
+
+      # Handle parentheses/braces - these start grouped values
+      if current_char == "(" || current_char == "{"
+        return # Let the main loop handle these
+      end
+
+      value = ""
+      while @position < @input.length
+        char = current_char
+        break if /[\s():{}]/.match?(char)
+        # Include hyphens in operator values (unlike regular words)
+
+        if char == "\\"
+          advance
+          if @position < @input.length
+            next_char = current_char
+            value += case next_char
+            when '"', "\\"
+              next_char
+            else
+              "\\" + next_char
+            end
+            advance
+          end
+        else
+          value += char
+          advance
+        end
+      end
+
+      return if value.empty?
+
+      # Classify the value
+      if /@/.match?(value)
+        add_token(:email, value)
+      elsif /^\d+$/.match?(value)
+        add_token(:number, value.to_i)
+      elsif value =~ /^\d{4}\/\d{2}\/\d{2}$/ || value =~ /^\d{2}\/\d{2}\/\d{4}$/
+        add_token(:date, value)
+      elsif /^(\d+)([dmy])$/.match?(value)
+        add_token(:relative_time, value)
+      else
+        add_token(:word, value)
+      end
     end
 
     def read_quoted_string

data/lib/gmail_search_syntax/version.rb

@@ -1,3 +1,3 @@
 module GmailSearchSyntax
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end

data/slop/EMBEDDED_HYPHENS.md

@@ -0,0 +1,102 @@
+# Embedded Hyphens in Gmail Search
+
+## Gmail's Actual Behavior
+
+Gmail treats hyphens differently depending on whether they are preceded by whitespace:
+
+### Embedded Hyphen (No Preceding Whitespace)
+
+When a hyphen appears immediately after a word character (no space before it), Gmail treats it as a **word separator**, not a negation operator. Both parts become separate search tokens that are implicitly ANDed together.
+
+```
+Coxlee-Gammage
+```
+
+Gmail behavior: Search for messages containing both "Coxlee" AND "Gammage". Both tokens get highlighted in search results.
+
+Parsed as:
+```ruby
+GmailSearchSyntax.parse!("Coxlee-Gammage")
+# => #<And [#<StringToken "Coxlee">, #<StringToken "Gammage">]>
+```
+
+### Space + Hyphen (Negation)
+
+When a hyphen is preceded by whitespace (or at the start of input), it functions as the **negation operator**.
+
+```
+Coxlee -Gammage
+```
+
+Gmail behavior: Search for messages containing "Coxlee" but NOT "Gammage".
+
+Parsed as:
+```ruby
+GmailSearchSyntax.parse!("Coxlee -Gammage")
+# => #<And [#<StringToken "Coxlee">, #<Not #<StringToken "Gammage">>]>
+```
+
+## Examples
+
+| Query | Parsed As | Meaning |
+|-------|-----------|---------|
+| `some-outfit` | `some AND outfit` | Contains both "some" and "outfit" |
+| `some -outfit` | `some AND NOT outfit` | Contains "some" but not "outfit" |
+| `a-b-c` | `a AND b AND c` | Contains all three tokens |
+| `-spam` | `NOT spam` | Does not contain "spam" |
+| `cats-dogs -birds` | `cats AND dogs AND NOT birds` | Contains "cats" and "dogs", not "birds" |
+
+## Real-World Use Cases
+
+### Hyphenated Names
+```
+from:Mary-Jane
+```
+Searches for emails where "from" contains "Mary" AND message contains "Jane".
+
+### Hyphenated Terms
+```
+self-service
+```
+Finds messages containing both "self" and "service".
+
+### Compound Words
+```
+e-commerce
+```
+Finds messages containing both "e" and "commerce".
+
+## Implementation Details
+
+The fix is in the tokenizer (`lib/gmail_search_syntax/tokenizer.rb`). When encountering a `-` character:
+
+1. Check if there's a non-whitespace character following (potential negation or word separator)
+2. Check if there's whitespace (or nothing) preceding the hyphen
+3. If preceded by whitespace or at start of input: treat as negation operator (`:minus` token)
+4. If preceded by non-whitespace: skip the hyphen (acts as word separator, no token emitted)
+
+```ruby
+when "-"
+  next_char = peek_char
+  prev_char = @position > 0 ? @input[@position - 1] : nil
+
+  if next_char && next_char !~ /\s/ && (prev_char.nil? || prev_char =~ /\s/)
+    # Negation: preceded by whitespace or at start, followed by non-whitespace
+    add_token(:minus, char)
+    advance
+  elsif prev_char && prev_char !~ /\s/
+    # Embedded hyphen: preceded by non-whitespace - skip as word separator
+    advance
+  else
+    read_word
+  end
+```
+
+## Bug That Was Fixed
+
+Previously, the gem incorrectly treated all hyphens followed by non-whitespace as negation:
+
+- **Old (incorrect):** `some-outfit` was parsed as `some AND NOT outfit`
+- **New (correct):** `some-outfit` is parsed as `some AND outfit`
+
+This matches Gmail's actual search behavior where hyphenated terms find messages containing both parts of the hyphenated word.

data/test/gmail_search_syntax_test.rb

@@ -15,7 +15,7 @@ class GmailSearchSyntaxTest < Minitest::Test
               # This is an Operator specification
               assert_operator(expected_operand, actual_value[index])
             elsif expected_operand.key?(:value)
-              # This is a StringToken specification
+              # This is a LooseWord specification
               assert_string_token(expected_operand, actual_value[index])
             else
               # Generic node assertion
@@ -34,10 +34,10 @@ class GmailSearchSyntaxTest < Minitest::Test
   end
 
   def assert_string_token(expected_properties, actual_string_token)
-    assert_instance_of StringToken, actual_string_token, "Expected StringToken, got #{actual_string_token.class}"
+    assert_instance_of LooseWord, actual_string_token, "Expected LooseWord, got #{actual_string_token.class}"
     expected_properties.each do |property, expected_value|
       actual_value = actual_string_token.public_send(property)
-      assert_equal expected_value, actual_value, "StringToken: expected #{property} to be #{expected_value.inspect}, got #{actual_value.inspect}"
+      assert_equal expected_value, actual_value, "LooseWord: expected #{property} to be #{expected_value.inspect}, got #{actual_value.inspect}"
     end
   end
 
@@ -134,29 +134,122 @@ class GmailSearchSyntaxTest < Minitest::Test
     assert_instance_of And, ast
 
     assert_equal 2, ast.operands.length
-    assert_instance_of StringToken, ast.operands[0]
+    assert_instance_of LooseWord, ast.operands[0]
     assert_equal "dinner", ast.operands[0].value
 
     assert_instance_of Not, ast.operands[1]
-    assert_instance_of StringToken, ast.operands[1].child
+    assert_instance_of LooseWord, ast.operands[1].child
     assert_equal "movie", ast.operands[1].child.value
   end
 
+  # Gmail behavior: embedded hyphens (no preceding whitespace) are word separators, not negation
+  # "Coxlee-Gammage" → Coxlee AND Gammage (both tokens highlighted)
+  # "Coxlee -Gammage" → Coxlee AND NOT Gammage (space+hyphen = negation)
+
+  def test_embedded_hyphen_is_word_separator
+    # Gmail behavior: hyphen without preceding whitespace separates words, not negation
+    ast = GmailSearchSyntax.parse!("some-outfit")
+    assert_instance_of And, ast
+
+    assert_equal 2, ast.operands.length
+    assert_instance_of LooseWord, ast.operands[0]
+    assert_equal "some", ast.operands[0].value
+
+    assert_instance_of LooseWord, ast.operands[1]
+    assert_equal "outfit", ast.operands[1].value
+  end
+
+  def test_embedded_hyphen_multiple
+    # Multiple hyphens: a-b-c → a AND b AND c
+    ast = GmailSearchSyntax.parse!("a-b-c")
+    assert_instance_of And, ast
+
+    assert_equal 3, ast.operands.length
+    assert_equal "a", ast.operands[0].value
+    assert_equal "b", ast.operands[1].value
+    assert_equal "c", ast.operands[2].value
+  end
+
+  def test_embedded_hyphen_real_name
+    # Real-world case: hyphenated names
+    ast = GmailSearchSyntax.parse!("Coxlee-Gammage")
+    assert_instance_of And, ast
+
+    assert_equal 2, ast.operands.length
+    assert_equal "Coxlee", ast.operands[0].value
+    assert_equal "Gammage", ast.operands[1].value
+  end
+
+  def test_space_hyphen_is_negation
+    # Space + hyphen = negation (unchanged behavior)
+    ast = GmailSearchSyntax.parse!("cats -dogs")
+    assert_instance_of And, ast
+
+    assert_equal 2, ast.operands.length
+    assert_instance_of LooseWord, ast.operands[0]
+    assert_equal "cats", ast.operands[0].value
+
+    assert_instance_of Not, ast.operands[1]
+    assert_equal "dogs", ast.operands[1].child.value
+  end
+
+  def test_embedded_hyphen_combined_with_negation
+    # Mixed: embedded hyphen + space-preceded negation
+    ast = GmailSearchSyntax.parse!("some-outfit -dogs")
+    assert_instance_of And, ast
+
+    assert_equal 3, ast.operands.length
+    assert_equal "some", ast.operands[0].value
+    assert_equal "outfit", ast.operands[1].value
+    assert_instance_of Not, ast.operands[2]
+    assert_equal "dogs", ast.operands[2].child.value
+  end
+
+  def test_negation_at_start_of_input
+    # Negation at start of input still works
+    ast = GmailSearchSyntax.parse!("-spam")
+    assert_instance_of Not, ast
+    assert_equal "spam", ast.child.value
+  end
+
+  def test_embedded_hyphen_with_operator
+    # Gmail keeps hyphens in operator values: from:mary-jane → operator with value "mary-jane"
+    ast = GmailSearchSyntax.parse!("from:mary-jane")
+    assert_instance_of Operator, ast
+    assert_equal "from", ast.name
+    assert_equal "mary-jane", ast.value
+  end
+
+  def test_hyphenated_label_name
+    # Common case: label names with hyphens
+    ast = GmailSearchSyntax.parse!("label:work-projects")
+    assert_instance_of Operator, ast
+    assert_equal "label", ast.name
+    assert_equal "work-projects", ast.value
+  end
+
+  def test_multiple_hyphens_in_operator_value
+    ast = GmailSearchSyntax.parse!("label:starter-league-thisforthat")
+    assert_instance_of Operator, ast
+    assert_equal "label", ast.name
+    assert_equal "starter-league-thisforthat", ast.value
+  end
+
   def test_around_operator
     ast = GmailSearchSyntax.parse!("holiday AROUND 10 vacation")
     assert_instance_of Around, ast
 
-    assert_instance_of StringToken, ast.left
+    assert_instance_of LooseWord, ast.left
     assert_equal "holiday", ast.left.value
     assert_equal 10, ast.distance
 
-    assert_instance_of StringToken, ast.right
+    assert_instance_of LooseWord, ast.right
     assert_equal "vacation", ast.right.value
   end
 
   def test_around_with_quoted_string
     ast = GmailSearchSyntax.parse!('"secret AROUND 25 birthday"')
-    assert_instance_of Substring, ast
+    assert_instance_of ExactWord, ast
     assert_equal "secret AROUND 25 birthday", ast.value
   end
 
@@ -187,7 +280,7 @@ class GmailSearchSyntaxTest < Minitest::Test
 
   def test_quoted_exact_phrase
     ast = GmailSearchSyntax.parse!('"dinner and movie tonight"')
-    assert_instance_of Substring, ast
+    assert_instance_of ExactWord, ast
     assert_equal "dinner and movie tonight", ast.value
   end
 
@@ -296,7 +389,7 @@ class GmailSearchSyntaxTest < Minitest::Test
 
   def test_plain_text_search
     ast = GmailSearchSyntax.parse!("meeting")
-    assert_instance_of StringToken, ast
+    assert_instance_of LooseWord, ast
     assert_equal "meeting", ast.value
   end
 
@@ -305,10 +398,10 @@ class GmailSearchSyntaxTest < Minitest::Test
     assert_instance_of And, ast
 
     assert_equal 2, ast.operands.length
-    assert_instance_of StringToken, ast.operands[0]
+    assert_instance_of LooseWord, ast.operands[0]
     assert_equal "project", ast.operands[0].value
 
-    assert_instance_of StringToken, ast.operands[1]
+    assert_instance_of LooseWord, ast.operands[1]
     assert_equal "report", ast.operands[1].value
   end
 
@@ -386,7 +479,7 @@ class GmailSearchSyntaxTest < Minitest::Test
 
   def test_quoted_string_with_operators_inside
     ast = GmailSearchSyntax.parse!('"from:amy to:bob"')
-    assert_instance_of Substring, ast
+    assert_instance_of ExactWord, ast
     assert_equal "from:amy to:bob", ast.value
   end
 
@@ -423,7 +516,7 @@ class GmailSearchSyntaxTest < Minitest::Test
 
   def test_parentheses_with_single_term
     ast = GmailSearchSyntax.parse!("(meeting)")
-    assert_instance_of StringToken, ast
+    assert_instance_of LooseWord, ast
     assert_equal "meeting", ast.value
   end
 
@@ -441,10 +534,10 @@ class GmailSearchSyntaxTest < Minitest::Test
     assert_instance_of And, ast
 
     assert_equal 2, ast.operands.length
-    assert_instance_of StringToken, ast.operands[0]
+    assert_instance_of LooseWord, ast.operands[0]
     assert_equal "meeting", ast.operands[0].value
 
-    assert_instance_of StringToken, ast.operands[1]
+    assert_instance_of LooseWord, ast.operands[1]
     assert_equal "project", ast.operands[1].value
   end
 
@@ -537,7 +630,7 @@ class GmailSearchSyntaxTest < Minitest::Test
     assert_equal 2, ast.value.operands[0].operands.length
     assert_equal "urgent", ast.value.operands[0].operands[0].value
     assert_equal "important", ast.value.operands[0].operands[1].value
-    assert_instance_of StringToken, ast.value.operands[1]
+    assert_instance_of LooseWord, ast.value.operands[1]
     assert_equal "meeting", ast.value.operands[1].value
   end
 
@@ -619,13 +712,13 @@ class GmailSearchSyntaxTest < Minitest::Test
 
   def test_quoted_string_with_escaped_quotes
     ast = GmailSearchSyntax.parse!('"She said \\"hello\\" to me"')
-    assert_instance_of Substring, ast
+    assert_instance_of ExactWord, ast
     assert_equal 'She said "hello" to me', ast.value
   end
 
   def test_quoted_string_with_escaped_backslash
     ast = GmailSearchSyntax.parse!('"path\\\\to\\\\file"')
-    assert_instance_of Substring, ast
+    assert_instance_of ExactWord, ast
     assert_equal 'path\\to\\file', ast.value
   end
 
@@ -638,13 +731,13 @@ class GmailSearchSyntaxTest < Minitest::Test
 
   def test_unquoted_text_with_escaped_quote
     ast = GmailSearchSyntax.parse!('meeting\\"room')
-    assert_instance_of StringToken, ast
+    assert_instance_of LooseWord, ast
     assert_equal 'meeting"room', ast.value
   end
 
   def test_unquoted_text_with_escaped_backslash
     ast = GmailSearchSyntax.parse!('path\\\\to\\\\file')
-    assert_instance_of StringToken, ast
+    assert_instance_of LooseWord, ast
     assert_equal 'path\\to\\file', ast.value
   end
 
@@ -679,7 +772,7 @@ class GmailSearchSyntaxTest < Minitest::Test
     assert_equal "Cora/Google", ast.operands[0].value
 
     # "Drive" becomes a separate search term
-    assert_instance_of StringToken, ast.operands[1]
+    assert_instance_of LooseWord, ast.operands[1]
     assert_equal "Drive", ast.operands[1].value
 
     # Second operator parsed correctly
@@ -762,4 +855,55 @@ class GmailSearchSyntaxTest < Minitest::Test
     # Second operator: label with "Notes"
     assert_operator({name: "label", value: "Notes"}, ast.operands[2])
   end
+
+  def test_label_with_consecutive_hyphens
+    # Labels like "Every (Every.To)" get sanitized to "every--every.to-"
+    # Gmail replaces parens, spaces, slashes with hyphens
+    ast = GmailSearchSyntax.parse!("label:every--every.to-")
+    assert_instance_of Operator, ast
+    assert_equal "label", ast.name
+    assert_equal "every--every.to-", ast.value
+  end
+
+  def test_label_with_dots_and_hyphens
+    # Complex label names with dots should work
+    ast = GmailSearchSyntax.parse!("label:cora-every--every.to-")
+    assert_instance_of Operator, ast
+    assert_equal "label", ast.name
+    assert_equal "cora-every--every.to-", ast.value
+  end
+
+  def test_quoted_word_with_or_group
+    # Quoted word followed by OR group in parentheses
+    ast = GmailSearchSyntax.parse!('"dropbox" (file OR share OR sync OR storage OR cloud)')
+    assert_instance_of And, ast
+    assert_equal 2, ast.operands.length
+
+    # First operand: quoted "dropbox" as ExactWord
+    assert_instance_of ExactWord, ast.operands[0]
+    assert_equal "dropbox", ast.operands[0].value
+
+    # Second operand: OR group with 5 terms
+    assert_instance_of Or, ast.operands[1]
+    assert_equal 5, ast.operands[1].operands.length
+    assert_equal "file", ast.operands[1].operands[0].value
+    assert_equal "share", ast.operands[1].operands[1].value
+    assert_equal "sync", ast.operands[1].operands[2].value
+    assert_equal "storage", ast.operands[1].operands[3].value
+    assert_equal "cloud", ast.operands[1].operands[4].value
+  end
+
+  def test_unknown_operator_is_searched_as_typed
+    assert_equal LooseWord.new("foo:bar"), GmailSearchSyntax.parse!("foo:bar")
+    assert_equal And.new([LooseWord.new("foo:bar"), LooseWord.new("baz")]), GmailSearchSyntax.parse!("foo:bar baz")
+    assert_equal LooseWord.new("http://x.y/z"), GmailSearchSyntax.parse!("http://x.y/z")
+    assert_equal LooseWord.new("a:b:c"), GmailSearchSyntax.parse!("a:b:c")
+    assert_equal And.new([LooseWord.new("foo"), LooseWord.new("bar")]), GmailSearchSyntax.parse!("foo: bar")
+    assert_equal And.new([Operator.new("from", "bob"), LooseWord.new("foo:bar")]), GmailSearchSyntax.parse!("from:bob foo:bar")
+  end
+
+  def test_a_stray_token_does_not_swallow_the_rest_of_the_query
+    assert_equal And.new([LooseWord.new("foo"), Operator.new("from", "bob")]), GmailSearchSyntax.parse!("foo ) from:bob")
+    assert_equal And.new([LooseWord.new("foo"), LooseWord.new("bar"), Operator.new("to", "ann")]), GmailSearchSyntax.parse!("foo : bar to:ann")
+  end
 end

data/test/sql_visitor_test.rb

@@ -255,7 +255,7 @@ class SqlVisitorTest < Minitest::Test
   def test_quoted_text_search_uses_substring
     sql, params = parse_and_visit('"meeting"')
 
-    # Quoted strings create Substring nodes which use LIKE %value%
+    # Quoted strings create ExactWord nodes which use LIKE %value%
     assert_includes sql, "m0.subject LIKE ?"
     assert_includes sql, "m0.body LIKE ?"
     assert_equal ["%meeting%", "%meeting%"], params

data/test/tokenizer_test.rb

@@ -94,6 +94,64 @@ class TokenizerTest < Minitest::Test
     assert_token_stream(expected, tokens)
   end
 
+  def test_tokenize_embedded_hyphen
+    # Gmail behavior: embedded hyphen (no preceding whitespace) is a word separator, not negation
+    tokens = tokenize("some-outfit")
+    expected = [
+      {type: :word, value: "some"},
+      {type: :word, value: "outfit"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
+  def test_tokenize_multiple_embedded_hyphens
+    # Multiple hyphens: a-b-c → three separate words
+    tokens = tokenize("a-b-c")
+    expected = [
+      {type: :word, value: "a"},
+      {type: :word, value: "b"},
+      {type: :word, value: "c"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
+  def test_tokenize_hyphenated_name
+    # Real-world case: hyphenated names like "Coxlee-Gammage"
+    tokens = tokenize("Coxlee-Gammage")
+    expected = [
+      {type: :word, value: "Coxlee"},
+      {type: :word, value: "Gammage"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
+  def test_tokenize_negation_at_start
+    # Negation at start of input
+    tokens = tokenize("-spam")
+    expected = [
+      {type: :minus},
+      {type: :word, value: "spam"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
+  def test_tokenize_embedded_hyphen_vs_negation
+    # Mixed: embedded hyphen + space-preceded negation
+    tokens = tokenize("some-outfit -dogs")
+    expected = [
+      {type: :word, value: "some"},
+      {type: :word, value: "outfit"},
+      {type: :minus},
+      {type: :word, value: "dogs"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
   def test_tokenize_around
     tokens = tokenize("holiday AROUND 10 vacation")
     expected = [

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gmail_search_syntax
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - me@julik.nl
@@ -92,6 +92,7 @@ files:
 - lib/gmail_search_syntax/tokenizer.rb
 - lib/gmail_search_syntax/version.rb
 - slop/ARCHITECTURE.md
+- slop/EMBEDDED_HYPHENS.md
 - slop/GMAIL_BEHAVIOR_COMPARISON.md
 - slop/GMAIL_COMPATIBILITY_COMPLETE.md
 - slop/GREEDY_VS_NON_GREEDY_TOKENIZATION.md
@@ -120,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Gmail search syntax parser
 test_files: []