gmail_search_syntax 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7c192efdd70d31b0747a356b1adab4c50160afb81e747d1088f2727f97c7262
-  data.tar.gz: 73f6401e73d2ecae59b6e6741c08786d2da3dffbbdcb6f21095c63bf40761f6d
+  metadata.gz: 47b56ee5467d6808c4ceae00289bf291551374a3e5854ed5223a5b2f5ca2f9ac
+  data.tar.gz: 27aa483d8296eb2a3e775aecfaeffc0813f60936cd9091e976eb5159559cafc9
 SHA512:
-  metadata.gz: ce609cacd2a6276d9178276ee84e0ccffc80eb8840bd62e6402e2275817ba2052559f903a55648373e87a79527415d6be4df1d4052429c83e3faa9abc1fe2c1c
-  data.tar.gz: 7badbc6024a67752bfab201577ebeb900ba106ed9979110c04eed6c76d35e33e26a7b1147927a2fd9d329f6a7bc28b71d2487d0f26e87f23a9dd7f867fac7e09
+  metadata.gz: 0170d5419e8ab3335a3bd4494f09c3b6e6f5a143cf26791a383f2189a9e12fcd49a596e56555670d018a6ffdffa13b13eed27e554cbe5ff82a221dcd12ed4bd7
+  data.tar.gz: 95d48f2e6aedb4160634db949e32be25823f8b808e71dc113f3ce9e8490b507e136a96372c329f8ff03f2476bc52f4dd31fc89f67a9ba2b7e11e5e0bcebf241f

data/lib/gmail_search_syntax/tokenizer.rb

@@ -55,9 +55,16 @@ module GmailSearchSyntax
           advance
         when "-"
           next_char = peek_char
-          if next_char && next_char !~ /\s/
+          prev_char = (@position > 0) ? @input[@position - 1] : nil
+          # Negation requires: non-whitespace follows AND (start of input OR whitespace precedes)
+          # Gmail behavior: "Coxlee-Gammage" → Coxlee AND Gammage (hyphen is word separator)
+          #                 "Coxlee -Gammage" → Coxlee AND NOT Gammage (space+hyphen = negation)
+          if next_char && next_char !~ /\s/ && (prev_char.nil? || prev_char =~ /\s/)
             add_token(:minus, char)
             advance
+          elsif prev_char && prev_char !~ /\s/
+            # Embedded hyphen (preceded by non-whitespace) - skip it as word separator
+            advance
           else
             read_word
           end

data/lib/gmail_search_syntax/version.rb

@@ -1,3 +1,3 @@
 module GmailSearchSyntax
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/slop/EMBEDDED_HYPHENS.md

@@ -0,0 +1,102 @@
+# Embedded Hyphens in Gmail Search
+
+## Gmail's Actual Behavior
+
+Gmail treats hyphens differently depending on whether they are preceded by whitespace:
+
+### Embedded Hyphen (No Preceding Whitespace)
+
+When a hyphen appears immediately after a word character (no space before it), Gmail treats it as a **word separator**, not a negation operator. Both parts become separate search tokens that are implicitly ANDed together.
+
+```
+Coxlee-Gammage
+```
+
+Gmail behavior: Search for messages containing both "Coxlee" AND "Gammage". Both tokens get highlighted in search results.
+
+Parsed as:
+```ruby
+GmailSearchSyntax.parse!("Coxlee-Gammage")
+# => #<And [#<StringToken "Coxlee">, #<StringToken "Gammage">]>
+```
+
+### Space + Hyphen (Negation)
+
+When a hyphen is preceded by whitespace (or at the start of input), it functions as the **negation operator**.
+
+```
+Coxlee -Gammage
+```
+
+Gmail behavior: Search for messages containing "Coxlee" but NOT "Gammage".
+
+Parsed as:
+```ruby
+GmailSearchSyntax.parse!("Coxlee -Gammage")
+# => #<And [#<StringToken "Coxlee">, #<Not #<StringToken "Gammage">>]>
+```
+
+## Examples
+
+| Query | Parsed As | Meaning |
+|-------|-----------|---------|
+| `some-outfit` | `some AND outfit` | Contains both "some" and "outfit" |
+| `some -outfit` | `some AND NOT outfit` | Contains "some" but not "outfit" |
+| `a-b-c` | `a AND b AND c` | Contains all three tokens |
+| `-spam` | `NOT spam` | Does not contain "spam" |
+| `cats-dogs -birds` | `cats AND dogs AND NOT birds` | Contains "cats" and "dogs", not "birds" |
+
+## Real-World Use Cases
+
+### Hyphenated Names
+```
+from:Mary-Jane
+```
+Searches for emails where "from" contains "Mary" AND message contains "Jane".
+
+### Hyphenated Terms
+```
+self-service
+```
+Finds messages containing both "self" and "service".
+
+### Compound Words
+```
+e-commerce
+```
+Finds messages containing both "e" and "commerce".
+
+## Implementation Details
+
+The fix is in the tokenizer (`lib/gmail_search_syntax/tokenizer.rb`). When encountering a `-` character:
+
+1. Check if there's a non-whitespace character following (potential negation or word separator)
+2. Check if there's whitespace (or nothing) preceding the hyphen
+3. If preceded by whitespace or at start of input: treat as negation operator (`:minus` token)
+4. If preceded by non-whitespace: skip the hyphen (acts as word separator, no token emitted)
+
+```ruby
+when "-"
+  next_char = peek_char
+  prev_char = @position > 0 ? @input[@position - 1] : nil
+
+  if next_char && next_char !~ /\s/ && (prev_char.nil? || prev_char =~ /\s/)
+    # Negation: preceded by whitespace or at start, followed by non-whitespace
+    add_token(:minus, char)
+    advance
+  elsif prev_char && prev_char !~ /\s/
+    # Embedded hyphen: preceded by non-whitespace - skip as word separator
+    advance
+  else
+    read_word
+  end
+```
+
+## Bug That Was Fixed
+
+Previously, the gem incorrectly treated all hyphens followed by non-whitespace as negation:
+
+- **Old (incorrect):** `some-outfit` was parsed as `some AND NOT outfit`
+- **New (correct):** `some-outfit` is parsed as `some AND outfit`
+
+This matches Gmail's actual search behavior where hyphenated terms find messages containing both parts of the hyphenated word.

data/test/gmail_search_syntax_test.rb

@@ -142,6 +142,91 @@ class GmailSearchSyntaxTest < Minitest::Test
     assert_equal "movie", ast.operands[1].child.value
   end
 
+  # Gmail behavior: embedded hyphens (no preceding whitespace) are word separators, not negation
+  # "Coxlee-Gammage" → Coxlee AND Gammage (both tokens highlighted)
+  # "Coxlee -Gammage" → Coxlee AND NOT Gammage (space+hyphen = negation)
+
+  def test_embedded_hyphen_is_word_separator
+    # Gmail behavior: hyphen without preceding whitespace separates words, not negation
+    ast = GmailSearchSyntax.parse!("some-outfit")
+    assert_instance_of And, ast
+
+    assert_equal 2, ast.operands.length
+    assert_instance_of StringToken, ast.operands[0]
+    assert_equal "some", ast.operands[0].value
+
+    assert_instance_of StringToken, ast.operands[1]
+    assert_equal "outfit", ast.operands[1].value
+  end
+
+  def test_embedded_hyphen_multiple
+    # Multiple hyphens: a-b-c → a AND b AND c
+    ast = GmailSearchSyntax.parse!("a-b-c")
+    assert_instance_of And, ast
+
+    assert_equal 3, ast.operands.length
+    assert_equal "a", ast.operands[0].value
+    assert_equal "b", ast.operands[1].value
+    assert_equal "c", ast.operands[2].value
+  end
+
+  def test_embedded_hyphen_real_name
+    # Real-world case: hyphenated names
+    ast = GmailSearchSyntax.parse!("Coxlee-Gammage")
+    assert_instance_of And, ast
+
+    assert_equal 2, ast.operands.length
+    assert_equal "Coxlee", ast.operands[0].value
+    assert_equal "Gammage", ast.operands[1].value
+  end
+
+  def test_space_hyphen_is_negation
+    # Space + hyphen = negation (unchanged behavior)
+    ast = GmailSearchSyntax.parse!("cats -dogs")
+    assert_instance_of And, ast
+
+    assert_equal 2, ast.operands.length
+    assert_instance_of StringToken, ast.operands[0]
+    assert_equal "cats", ast.operands[0].value
+
+    assert_instance_of Not, ast.operands[1]
+    assert_equal "dogs", ast.operands[1].child.value
+  end
+
+  def test_embedded_hyphen_combined_with_negation
+    # Mixed: embedded hyphen + space-preceded negation
+    ast = GmailSearchSyntax.parse!("some-outfit -dogs")
+    assert_instance_of And, ast
+
+    assert_equal 3, ast.operands.length
+    assert_equal "some", ast.operands[0].value
+    assert_equal "outfit", ast.operands[1].value
+    assert_instance_of Not, ast.operands[2]
+    assert_equal "dogs", ast.operands[2].child.value
+  end
+
+  def test_negation_at_start_of_input
+    # Negation at start of input still works
+    ast = GmailSearchSyntax.parse!("-spam")
+    assert_instance_of Not, ast
+    assert_equal "spam", ast.child.value
+  end
+
+  def test_embedded_hyphen_with_operator
+    # Embedded hyphen in operator context
+    ast = GmailSearchSyntax.parse!("from:mary-jane")
+    assert_instance_of And, ast
+
+    # "from:mary" becomes operator, "-jane" is embedded hyphen → "jane" is separate word
+    assert_equal 2, ast.operands.length
+    assert_instance_of Operator, ast.operands[0]
+    assert_equal "from", ast.operands[0].name
+    assert_equal "mary", ast.operands[0].value
+
+    assert_instance_of StringToken, ast.operands[1]
+    assert_equal "jane", ast.operands[1].value
+  end
+
   def test_around_operator
     ast = GmailSearchSyntax.parse!("holiday AROUND 10 vacation")
     assert_instance_of Around, ast

data/test/tokenizer_test.rb

@@ -94,6 +94,64 @@ class TokenizerTest < Minitest::Test
     assert_token_stream(expected, tokens)
   end
 
+  def test_tokenize_embedded_hyphen
+    # Gmail behavior: embedded hyphen (no preceding whitespace) is a word separator, not negation
+    tokens = tokenize("some-outfit")
+    expected = [
+      {type: :word, value: "some"},
+      {type: :word, value: "outfit"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
+  def test_tokenize_multiple_embedded_hyphens
+    # Multiple hyphens: a-b-c → three separate words
+    tokens = tokenize("a-b-c")
+    expected = [
+      {type: :word, value: "a"},
+      {type: :word, value: "b"},
+      {type: :word, value: "c"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
+  def test_tokenize_hyphenated_name
+    # Real-world case: hyphenated names like "Coxlee-Gammage"
+    tokens = tokenize("Coxlee-Gammage")
+    expected = [
+      {type: :word, value: "Coxlee"},
+      {type: :word, value: "Gammage"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
+  def test_tokenize_negation_at_start
+    # Negation at start of input
+    tokens = tokenize("-spam")
+    expected = [
+      {type: :minus},
+      {type: :word, value: "spam"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
+  def test_tokenize_embedded_hyphen_vs_negation
+    # Mixed: embedded hyphen + space-preceded negation
+    tokens = tokenize("some-outfit -dogs")
+    expected = [
+      {type: :word, value: "some"},
+      {type: :word, value: "outfit"},
+      {type: :minus},
+      {type: :word, value: "dogs"},
+      {type: :eof}
+    ]
+    assert_token_stream(expected, tokens)
+  end
+
   def test_tokenize_around
     tokens = tokenize("holiday AROUND 10 vacation")
     expected = [

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gmail_search_syntax
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - me@julik.nl
@@ -92,6 +92,7 @@ files:
 - lib/gmail_search_syntax/tokenizer.rb
 - lib/gmail_search_syntax/version.rb
 - slop/ARCHITECTURE.md
+- slop/EMBEDDED_HYPHENS.md
 - slop/GMAIL_BEHAVIOR_COMPARISON.md
 - slop/GMAIL_COMPATIBILITY_COMPLETE.md
 - slop/GREEDY_VS_NON_GREEDY_TOKENIZATION.md