gmail_search_syntax 0.1.0

data/examples/alias_collision_fix.rb

@@ -0,0 +1,43 @@
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "gmail_search_syntax"
+
+puts "=" * 80
+puts "Table Alias Uniqueness Fix"
+puts "=" * 80
+puts
+puts "When using operators with subqueries (OR, AND, NOT, etc.), each subquery"
+puts "needs its own visitor. Previously, each sub-visitor had its own alias"
+puts "counter starting at 0, causing alias collisions like:"
+puts
+puts "  WRONG: ... ma1 ... ma1 ... (same alias twice!)"
+puts
+puts "Now, all sub-visitors share a single counter ((1..).each enumerator), ensuring:"
+puts
+puts "  RIGHT: ... ma1 ... ma3 ... (unique aliases)"
+puts
+puts "=" * 80
+
+query = "from:alice@example.com OR from:bob@example.com"
+puts "\nQuery: #{query}"
+puts "-" * 80
+
+ast = GmailSearchSyntax.parse!(query)
+visitor = GmailSearchSyntax::SQLiteVisitor.new
+visitor.visit(ast)
+sql, _ = visitor.to_query.to_sql
+
+puts "\nGenerated SQL:"
+puts sql
+puts
+
+# Highlight the aliases
+ma1_count = sql.scan(/\bma1\b/).size
+ma3_count = sql.scan(/\bma3\b/).size
+
+puts "Alias usage:"
+puts "  ma1: appears #{ma1_count} times"
+puts "  ma3: appears #{ma3_count} times"
+puts
+puts "✓ No alias collision! Each JOIN has a unique alias."
+puts
+puts "=" * 80

data/examples/demo.rb

@@ -0,0 +1,28 @@
+require_relative "../lib/gmail_search_syntax"
+
+puts "Gmail Search Syntax Parser - Demo"
+puts "=" * 50
+puts
+
+queries = [
+  "from:amy@example.com",
+  "subject:meeting has:attachment",
+  "from:boss OR from:manager",
+  "{from:amy from:bob from:charlie}",
+  "dinner -movie",
+  "holiday AROUND 10 vacation",
+  'from:manager subject:"quarterly review" after:2024/01/01',
+  "is:unread label:important -label:spam",
+  "(from:team OR from:boss) subject:urgent",
+  "from:(mischa@ OR julik@) subject:meeting",
+  "to:(alice@ OR bob@ OR charlie@)",
+  "from:{mischa@ marc@}",
+  "from:{alice@ bob@} to:{charlie@ david@}"
+]
+
+queries.each do |query|
+  puts "Query: #{query}"
+  ast = GmailSearchSyntax.parse!(query)
+  puts "AST:   #{ast.inspect}"
+  puts
+end

data/examples/gmail_message_id_demo.rb

@@ -0,0 +1,118 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# This example demonstrates how Gmail message IDs encode timestamps
+# Based on: https://www.metaspike.com/dates-gmail-message-id-thread-id-timestamps/
+#
+# Gmail Message IDs are hexadecimal values where the first part (all but last 5 digits)
+# encodes the timestamp in milliseconds since epoch.
+
+require "time"
+
+def generate_gmail_message_id(time, random: Random.new)
+  # Get timestamp in milliseconds since epoch
+  timestamp_ms = (time.to_f * 1000).to_i
+  # Convert timestamp to hex and append first 5 hex digits from random bytes
+  timestamp_hex = timestamp_ms.to_s(16)
+  # Generate 3 random bytes (24 bits = 6 hex digits, we'll use first 5)
+  # Using bytes is much faster than generating individual hex digits
+  random_hex = random.bytes(3).unpack1("H*")[0, 5]
+
+  timestamp_hex + random_hex
+end
+
+def decode_gmail_message_id(message_id)
+  # Drop last 5 digits to get the timestamp part
+  timestamp_hex = message_id[0..-6]
+
+  # Convert from hex to milliseconds
+  timestamp_ms = timestamp_hex.to_i(16)
+
+  # Convert to Time object
+  Time.at(timestamp_ms / 1000.0)
+end
+
+puts "Gmail Message ID Generation Demo"
+puts "=" * 50
+puts
+
+# Example 1: Generate a message ID for the current time
+puts "Example 1: Current timestamp"
+current_time = Time.now
+message_id = generate_gmail_message_id(current_time)
+decoded_time = decode_gmail_message_id(message_id)
+
+puts "Original time:  #{current_time}"
+puts "Message ID:     #{message_id}"
+puts "Decoded time:   #{decoded_time}"
+puts "Match:          #{(current_time - decoded_time).abs < 0.001}"
+puts
+
+# Example 2: Examples from the article
+puts "Example 2: Known Gmail message IDs from the article"
+examples = {
+  "172ed79b0337c14f" => "Thursday, June 25, 2020 9:54:34.675 PM (UTC)",
+  "ffff3432161af8b" => "Wednesday, November 3, 2004 4:11:11.254 PM (UTC)"
+}
+
+examples.each do |msg_id, expected_date|
+  decoded = decode_gmail_message_id(msg_id)
+  puts "Message ID:     #{msg_id}"
+  puts "Expected:       #{expected_date}"
+  puts "Decoded:        #{decoded.utc}"
+  puts
+end
+
+# Example 3: Generate IDs for specific dates
+puts "Example 3: Generate IDs for specific dates"
+dates = [
+  Time.parse("2020-01-01 00:00:00 UTC"),
+  Time.parse("2023-06-15 12:30:45 UTC"),
+  Time.parse("2024-12-25 18:45:00 UTC")
+]
+
+dates.each do |date|
+  msg_id = generate_gmail_message_id(date)
+  decoded = decode_gmail_message_id(msg_id)
+  puts "Original:       #{date}"
+  puts "Message ID:     #{msg_id}"
+  puts "Decoded:        #{decoded}"
+  puts "Accurate:       #{(date - decoded).abs < 0.001}"
+  puts
+end
+
+# Example 4: Using a seeded Random for reproducible IDs (useful for testing)
+puts "Example 4: Reproducible IDs with seeded Random"
+test_time = Time.parse("2024-06-15 12:00:00 UTC")
+seeded_random = Random.new(12345)
+
+# Generate the same ID twice with the same seed
+id1 = generate_gmail_message_id(test_time, random: seeded_random)
+seeded_random = Random.new(12345) # Reset seed
+id2 = generate_gmail_message_id(test_time, random: seeded_random)
+
+puts "Time:           #{test_time}"
+puts "ID (seed 1):    #{id1}"
+puts "ID (seed 2):    #{id2}"
+puts "Reproducible:   #{id1 == id2}"
+puts
+
+# Generate different IDs with different seeds
+seeded_random1 = Random.new(11111)
+seeded_random2 = Random.new(22222)
+id_a = generate_gmail_message_id(test_time, random: seeded_random1)
+id_b = generate_gmail_message_id(test_time, random: seeded_random2)
+
+puts "ID (seed 11111): #{id_a}"
+puts "ID (seed 22222): #{id_b}"
+puts "Different:       #{id_a != id_b}"
+puts
+
+puts "=" * 50
+puts "Key Points:"
+puts "- Message IDs are 15-16 hexadecimal digits"
+puts "- First (n-5) digits encode timestamp in milliseconds"
+puts "- Last 5 digits are random for uniqueness"
+puts "- Overflow happened on Nov 3, 2004 (10 → 11 digits for timestamp)"
+puts "- Optional random: parameter allows reproducible testing"
+puts "- Uses bytes for ~10x faster random generation"

data/examples/postgres_vs_sqlite.rb

@@ -0,0 +1,55 @@
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "gmail_search_syntax"
+
+# This example demonstrates the difference between SQLite and PostgreSQL
+# SQL generation, particularly for relative date operators
+
+queries = [
+  "older_than:7d",
+  "newer_than:3m",
+  "from:alice@example.com older_than:1y",
+  "subject:meeting newer_than:2d"
+]
+
+queries.each do |query_string|
+  puts "\n" + "=" * 80
+  puts "Query: #{query_string}"
+  puts "=" * 80
+
+  ast = GmailSearchSyntax.parse!(query_string)
+
+  # SQLite version
+  puts "\n--- SQLite ---"
+  sqlite_visitor = GmailSearchSyntax::SQLiteVisitor.new(current_user_email: "me@example.com")
+  sqlite_visitor.visit(ast)
+  sqlite_sql, sqlite_params = sqlite_visitor.to_query.to_sql
+
+  puts "SQL:"
+  puts sqlite_sql
+  puts "\nParameters:"
+  sqlite_params.each_with_index do |param, idx|
+    puts "  #{idx + 1}. #{param.inspect}"
+  end
+
+  # PostgreSQL version
+  puts "\n--- PostgreSQL ---"
+  postgres_visitor = GmailSearchSyntax::PostgresVisitor.new(current_user_email: "me@example.com")
+  postgres_visitor.visit(ast)
+  postgres_sql, postgres_params = postgres_visitor.to_query.to_sql
+
+  puts "SQL:"
+  puts postgres_sql
+  puts "\nParameters:"
+  postgres_params.each_with_index do |param, idx|
+    puts "  #{idx + 1}. #{param.inspect}"
+  end
+end
+
+puts "\n" + "=" * 80
+puts "Key Differences:"
+puts "=" * 80
+puts "- SQLite uses: datetime('now', '-7 days')"
+puts "- PostgreSQL uses: NOW() - '7 days'::interval"
+puts "- The relative time parameter format differs:"
+puts "  * SQLite: '-7 days' (negative number)"
+puts "  * PostgreSQL: '7 days' (positive number with cast)"

data/examples/sql_query.rb

@@ -0,0 +1,47 @@
+require_relative "../lib/gmail_search_syntax"
+
+queries = [
+  "from:amy@example.com",
+  "from:@example.com",
+  "from:amy AND to:bob",
+  "subject:meeting has:attachment",
+  "label:important is:unread",
+  "after:2024/01/01 -from:spam@example.com",
+  'from:(amy@example.com OR bob@example.com) subject:"urgent meeting"',
+  "larger:10M filename:pdf",
+  "category:primary is:starred"
+]
+
+queries.each do |query_string|
+  puts "\n" + "=" * 80
+  puts "Query: #{query_string}"
+  puts "=" * 80
+
+  ast = GmailSearchSyntax.parse!(query_string)
+  visitor = GmailSearchSyntax::SqlVisitor.new(current_user_email: "me@example.com")
+  visitor.visit(ast)
+
+  sql, params = visitor.to_query.to_sql
+
+  puts "\nSQL:"
+  puts sql
+
+  puts "\nParameters:"
+  params.each_with_index do |param, idx|
+    puts "  #{idx + 1}. #{param.inspect}"
+  end
+end
+
+puts "\n" + "=" * 80
+puts "AROUND operator (generates no-op condition):"
+puts "=" * 80
+
+ast = GmailSearchSyntax.parse!("holiday AROUND 10 vacation")
+visitor = GmailSearchSyntax::SqlVisitor.new
+visitor.visit(ast)
+sql, _ = visitor.to_query.to_sql
+
+puts "\nSQL:"
+puts sql
+puts "\nParameters:"
+puts "  (none)"

data/lib/GMAIL_SEARCH_OPERATORS.md

@@ -0,0 +1,58 @@
+# Gmail Search Operators Reference
+
+Source: [Gmail Help - Refine searches in Gmail](https://support.google.com/mail/answer/7190?hl=en&co=GENIE.Platform%3DDesktop)
+
+You can use words or symbols called search operators to filter your Gmail search results. You can also combine operators to filter your results even more.
+
+## How to Use Search Operators
+
+1. On your computer, go to Gmail.
+2. At the top, click the search box.
+3. Enter a search operator.
+
+### Tips
+
+- After you search, you can use the results to set up a filter for these messages.
+- When using numbers as part of your query, a space or a dash (-) will separate a number while a dot (.) will be a decimal. For example, 01.2047-100 is considered 2 numbers: 01.2047 and 100.
+
+## Search Operators
+
+| Search Operator | Description | Example |
+|---|---|---|
+| `from:` | Find emails sent from a specific person. | `from:me`<br>`from:amy@example.com` |
+| `to:` | Find emails sent to a specific person. | `to:me`<br>`to:john@example.com` |
+| `cc:`<br>`bcc:` | Find emails that include specific people in the "Cc" or "Bcc" fields. | `cc:john@example.com`<br>`bcc:david@example.com` |
+| `subject:` | Find emails by a word or phrase in the subject line. | `subject:dinner`<br>`subject:anniversary party` |
+| `after:`<br>`before:`<br>`older:`<br>`newer:` | Search for emails received during a certain time period. | `after:2004/04/16`<br>`after:04/16/2004`<br>`before:2004/04/18`<br>`before:04/18/2004` |
+| `older_than:`<br>`newer_than:` | Search for emails older or newer than a time period. Use d (day), m (month), or y (year). | `older_than:1y`<br>`newer_than:2d` |
+| `OR` or `{ }` | Find emails that match one or more of your search criteria. | `from:amy OR from:david`<br>`{from:amy from:david}` |
+| `AND` | Find emails that match all of your search criteria. | `from:amy AND to:david` |
+| `-` | Exclude emails from your search criteria. | `dinner -movie` |
+| `AROUND` | Find emails with words near each other. Use the number to say how many words apart the words can be. Add quotes to find messages in which the word you put first stays first. | `holiday AROUND 10 vacation`<br>`"secret AROUND 25 birthday"` |
+| `label:` | Find emails under one of your labels. | `label:friends`<br>`label:important` |
+| `category:` | If you use inbox categories, find emails under one of the categories. | `category:primary`<br>`category:social`<br>`category:promotions`<br>`category:updates`<br>`category:forums`<br>`category:reservations`<br>`category:purchases` |
+| `has:` | Find emails that include:<br>- Attachments<br>- Inline images<br>- YouTube videos<br>- Drive files<br>- Google Docs<br>- Google Sheets<br>- Google Slides | `has:attachment`<br>`has:youtube`<br>`has:drive`<br>`has:document`<br>`has:spreadsheet`<br>`has:presentation` |
+| `list:` | Find emails from a mailing list. | `list:info@example.com` |
+| `filename:` | Find emails that have attachments with a certain name or file type. | `filename:pdf`<br>`filename:homework.txt` |
+| `" "` | Search for emails with an exact word or phrase. | `"dinner and movie tonight"` |
+| `( )` | Group multiple search terms together. | `subject:(dinner movie)` |
+| `in:anywhere` | Find emails across Gmail. This includes emails in Spam and Trash. | `in:anywhere movie` |
+| `in:archive` | Search for archived messages. | `in:archive payment reminder` |
+| `in:snoozed` | Find emails that you snoozed. | `in:snoozed birthday reminder` |
+| `is:muted` | Find emails that you muted. | `is:muted subject:team celebration` |
+| `is:` | Search for emails by their status:<br>- Important<br>- Starred<br>- Unread<br>- Read | `is:important`<br>`is:starred`<br>`is:unread`<br>`is:read` |
+| `has:yellow-star`<br>`has:orange-star`<br>`has:red-star`<br>`has:purple-star`<br>`has:blue-star`<br>`has:green-star`<br>`has:red-bang`<br>`has:orange-guillemet`<br>`has:yellow-bang`<br>`has:green-check`<br>`has:blue-info`<br>`has:purple-question` | If you set up different star options, you can search for emails under a star option. | `has:yellow-star OR has:purple-question` |
+| `deliveredto:` | Find emails delivered to a specific email address. | `deliveredto:username@example.com` |
+| `size:`<br>`larger:`<br>`smaller:` | Find emails by their size. | `size:1000000`<br>`larger:10M` |
+| `+` | Find emails that match a word exactly. | `+unicorn` |
+| `rfc822msgid:` | Find emails with a specific message-id header. | `rfc822msgid:200503292@example.com` |
+| `has:userlabels`<br>`has:nouserlabels` | Find emails that have or don't have a label. Labels are only added to a message, and not an entire conversation. | `has:userlabels`<br>`has:nouserlabels` |
+| `label:encryptedmail` | Find emails sent with Client-side encryption. | `label:encryptedmail` |
+
+## Additional Notes
+
+- You can combine operators to create complex searches
+- Operators work with both implicit AND (space-separated terms) and explicit AND
+- Use parentheses `()` or braces `{}` to group terms and create complex queries
+- Operator values can contain expressions, e.g., `from:(alice@ OR bob@)` or `from:{alice@ bob@}`
+

data/lib/gmail_search_syntax/ast.rb

@@ -0,0 +1,100 @@
+module GmailSearchSyntax
+  module AST
+    class Node
+      def ==(other)
+        self.class == other.class && attributes == other.attributes
+      end
+
+      def attributes
+        instance_variables.map { |var| instance_variable_get(var) }
+      end
+    end
+
+    class Operator < Node
+      attr_reader :name, :value
+
+      def initialize(name, value)
+        @name = name
+        @value = value
+      end
+
+      def inspect
+        "#<Operator #{@name}: #{@value.inspect}>"
+      end
+    end
+
+    class Text < Node
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+      end
+
+      def inspect
+        "#<Text #{@value.inspect}>"
+      end
+    end
+
+    class And < Node
+      attr_reader :operands
+
+      def initialize(operands)
+        @operands = operands
+      end
+
+      def inspect
+        "#<And #{@operands.map(&:inspect).join(" AND ")}>"
+      end
+    end
+
+    class Or < Node
+      attr_reader :operands
+
+      def initialize(operands)
+        @operands = operands
+      end
+
+      def inspect
+        "#<Or #{@operands.map(&:inspect).join(" OR ")}>"
+      end
+    end
+
+    class Not < Node
+      attr_reader :child
+
+      def initialize(child)
+        @child = child
+      end
+
+      def inspect
+        "#<Not #{@child.inspect}>"
+      end
+    end
+
+    class Group < Node
+      attr_reader :children
+
+      def initialize(children = [])
+        @children = children
+      end
+
+      def inspect
+        "#<Group #{@children.inspect}>"
+      end
+    end
+
+    class Around < Node
+      attr_reader :left, :distance, :right
+
+      def initialize(left, distance, right)
+        @left = left
+        @distance = distance
+        @right = right
+      end
+
+      def inspect
+        "#<Around #{@left.inspect} AROUND #{@distance} #{@right.inspect}>"
+      end
+    end
+  end
+end

data/lib/gmail_search_syntax/parser.rb

@@ -0,0 +1,224 @@
+module GmailSearchSyntax
+  class Parser
+    OPERATORS = %w[
+      from to cc bcc subject after before older newer older_than newer_than
+      label category has list filename in is deliveredto size larger smaller
+      rfc822msgid
+    ].freeze
+
+    def initialize(tokens)
+      @tokens = tokens
+      @position = 0
+    end
+
+    def parse!
+      children = []
+
+      until eof?
+        node = parse_expression
+        children << node if node
+      end
+
+      if children.empty?
+        raise GmailSearchSyntax::EmptyQueryError, "Query cannot be empty"
+      end
+
+      children.first
+    end
+
+    private
+
+    def current_token
+      @tokens[@position]
+    end
+
+    def peek_token(offset = 1)
+      @tokens[@position + offset]
+    end
+
+    def advance
+      @position += 1
+    end
+
+    def eof?
+      current_token.nil? || current_token.type == :eof
+    end
+
+    def parse_expression
+      parse_or_expression
+    end
+
+    def parse_or_expression
+      operands = [parse_and_expression]
+
+      while current_token&.type == :or
+        advance
+        operands << parse_and_expression
+      end
+
+      (operands.length == 1) ? operands.first : AST::Or.new(operands)
+    end
+
+    def parse_and_expression
+      operands = []
+
+      first = parse_around_expression
+      operands << first if first
+
+      while current_token&.type == :and
+        advance
+        operand = parse_around_expression
+        operands << operand if operand
+      end
+
+      while !eof? && current_token.type != :or && current_token.type != :rparen &&
+          current_token.type != :rbrace && current_token.type != :and
+        operand = parse_around_expression
+        break unless operand
+        operands << operand
+      end
+
+      return nil if operands.empty?
+      (operands.length == 1) ? operands.first : AST::And.new(operands)
+    end
+
+    def parse_around_expression
+      left = parse_unary_expression
+
+      if current_token&.type == :around
+        advance
+        distance = 5
+
+        if current_token&.type == :number
+          distance = current_token.value
+          advance
+        end
+
+        right = parse_unary_expression
+        return AST::Around.new(left, distance, right)
+      end
+
+      left
+    end
+
+    def parse_unary_expression
+      if current_token&.type == :minus
+        advance
+        child = parse_primary_expression
+        return AST::Not.new(child)
+      end
+
+      if current_token&.type == :plus
+        advance
+        return parse_primary_expression
+      end
+
+      parse_primary_expression
+    end
+
+    def parse_primary_expression
+      return nil if eof?
+
+      case current_token.type
+      when :lparen
+        parse_parentheses
+      when :lbrace
+        parse_braces
+      when :word
+        parse_operator_or_text
+      when :quoted_string
+        value = current_token.value
+        advance
+        AST::Text.new(value)
+      when :email, :number, :date, :relative_time
+        value = current_token.value
+        advance
+        AST::Text.new(value)
+      else
+        advance
+        nil
+      end
+    end
+
+    def parse_parentheses
+      advance
+
+      children = []
+      while !eof? && current_token.type != :rparen
+        node = parse_expression
+        children << node if node
+        break if current_token.type == :rparen
+      end
+
+      advance if current_token&.type == :rparen
+
+      (children.length == 1) ? children.first : AST::Group.new(children)
+    end
+
+    def parse_braces
+      advance
+
+      children = []
+      while !eof? && current_token.type != :rbrace
+        node = parse_unary_expression
+        children << node if node
+        break if current_token.type == :rbrace
+      end
+
+      advance if current_token&.type == :rbrace
+
+      (children.length == 1) ? children.first : AST::Or.new(children)
+    end
+
+    def parse_operator_or_text
+      word = current_token.value
+
+      if OPERATORS.include?(word.downcase) && peek_token&.type == :colon
+        operator_name = word.downcase
+        advance
+        advance
+
+        value = parse_operator_value
+        return AST::Operator.new(operator_name, value)
+      end
+
+      advance
+      AST::Text.new(word)
+    end
+
+    def parse_operator_value
+      return nil if eof?
+
+      case current_token.type
+      when :word
+        value = current_token.value
+        advance
+        value
+      when :email
+        value = current_token.value
+        advance
+        value
+      when :quoted_string
+        value = current_token.value
+        advance
+        value
+      when :number
+        value = current_token.value
+        advance
+        value
+      when :date
+        value = current_token.value
+        advance
+        value
+      when :relative_time
+        value = current_token.value
+        advance
+        value
+      when :lparen
+        parse_parentheses
+      when :lbrace
+        parse_braces
+      end
+    end
+  end
+end