sqs_processor 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +15 -47
- data/lib/sqs_processor/processor.rb +6 -21
- data/lib/sqs_processor/version.rb +1 -1
- metadata +2 -4
- data/bin/sqs_processor +0 -77
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b236be094b122ba859d742bff3fb3973e70cc9328dcd14244517ac09c4adec45
|
4
|
+
data.tar.gz: 32d709d43420508be56b438e6ae2958efd6ad5aeef23e1a4408fd296a87e0bb0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bc53c5db71dea861a64b1e282e7466ff9378fc664c0fa3bbe0c161189211f31d3ef925b1da122e4d1a29b8c4cb3fcdcc29acefc71ddc9342002cc6166ad22ba4
|
7
|
+
data.tar.gz: 208bb5ac50214e931dc0935a13d01cdac120460ad1c80a10a8fb060890a86cb07ad353d9730eff40f25d8724bb9d75db598a2564ae342de24837b4afbc6b4dac
|
data/README.md
CHANGED
@@ -58,37 +58,7 @@ DATA_SYNC_SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/123456789012/your-qu
|
|
58
58
|
|
59
59
|
## Usage
|
60
60
|
|
61
|
-
|
62
|
-
|
63
|
-
```bash
|
64
|
-
sqs_processor [options]
|
65
|
-
```
|
66
|
-
|
67
|
-
### Command Line Options
|
68
|
-
|
69
|
-
```bash
|
70
|
-
sqs_processor [options]
|
71
|
-
|
72
|
-
Options:
|
73
|
-
-q, --queue-url URL SQS Queue URL
|
74
|
-
-r, --region REGION AWS Region
|
75
|
-
-m, --max-messages NUMBER Max messages per batch (default: 10)
|
76
|
-
-v, --visibility-timeout SECONDS Visibility timeout in seconds (default: 30)
|
77
|
-
-h, --help Show this help message
|
78
|
-
```
|
79
|
-
|
80
|
-
### Examples
|
81
|
-
|
82
|
-
```bash
|
83
|
-
# Basic usage with environment variables
|
84
|
-
sqs_processor
|
85
|
-
|
86
|
-
# Specify queue URL and region
|
87
|
-
sqs_processor -q https://sqs.us-west-2.amazonaws.com/123456789012/my-queue -r us-west-2
|
88
|
-
|
89
|
-
# Custom batch size and visibility timeout
|
90
|
-
sqs_processor -m 5 -v 60
|
91
|
-
```
|
61
|
+
The gem is designed to be used programmatically in your Ruby applications.
|
92
62
|
|
93
63
|
### Programmatic Usage
|
94
64
|
|
@@ -97,14 +67,14 @@ sqs_processor -m 5 -v 60
|
|
97
67
|
```ruby
|
98
68
|
require 'sqs_processor'
|
99
69
|
|
100
|
-
# Create a processor instance
|
70
|
+
# Create a processor instance with AWS credentials
|
101
71
|
processor = SQSProcessor::Processor.new(
|
102
72
|
queue_url: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
|
103
|
-
region: 'us-east-1',
|
104
|
-
max_messages: 10,
|
105
|
-
visibility_timeout: 30,
|
106
73
|
aws_access_key_id: 'your-access-key',
|
107
|
-
aws_secret_access_key: 'your-secret-key'
|
74
|
+
aws_secret_access_key: 'your-secret-key',
|
75
|
+
aws_region: 'us-east-1',
|
76
|
+
max_messages: 10,
|
77
|
+
visibility_timeout: 30
|
108
78
|
)
|
109
79
|
|
110
80
|
# Start processing messages
|
@@ -117,8 +87,7 @@ The gem uses a hook-based approach for message processing. You must implement th
|
|
117
87
|
|
118
88
|
### Hook Method
|
119
89
|
|
120
|
-
The `handle_message(
|
121
|
-
- `message`: The SQS message object (contains message_id, receipt_handle, etc.)
|
90
|
+
The `handle_message(body)` method receives:
|
122
91
|
- `body`: The parsed JSON body of the message
|
123
92
|
|
124
93
|
Return `true` if processing was successful (message will be deleted from queue), or `false` if processing failed (message will remain in queue for retry).
|
@@ -141,8 +110,8 @@ To implement custom message processing logic, create a subclass of `SQSProcessor
|
|
141
110
|
require 'sqs_processor'
|
142
111
|
|
143
112
|
class MyCustomProcessor < SQSProcessor::Processor
|
144
|
-
def handle_message(
|
145
|
-
# This method receives the
|
113
|
+
def handle_message(body)
|
114
|
+
# This method receives the parsed message body
|
146
115
|
# Return true if processing was successful, false otherwise
|
147
116
|
|
148
117
|
case body['event_type']
|
@@ -174,9 +143,9 @@ end
|
|
174
143
|
# Usage
|
175
144
|
processor = MyCustomProcessor.new(
|
176
145
|
queue_url: 'your-queue-url',
|
177
|
-
region: 'us-east-1',
|
178
146
|
aws_access_key_id: 'your-access-key',
|
179
|
-
aws_secret_access_key: 'your-secret-key'
|
147
|
+
aws_secret_access_key: 'your-secret-key',
|
148
|
+
aws_region: 'us-east-1'
|
180
149
|
)
|
181
150
|
processor.process_messages
|
182
151
|
```
|
@@ -188,13 +157,12 @@ processor.process_messages
|
|
188
157
|
The `SQSProcessor::Processor.new` method accepts the following parameters:
|
189
158
|
|
190
159
|
- `queue_url:` (required) - The SQS queue URL
|
191
|
-
- `
|
160
|
+
- `aws_access_key_id:` (required) - AWS access key ID
|
161
|
+
- `aws_secret_access_key:` (required) - AWS secret access key
|
162
|
+
- `aws_region:` (optional, default: 'us-east-1') - AWS region
|
163
|
+
- `aws_session_token:` (optional) - AWS session token for temporary credentials
|
192
164
|
- `max_messages:` (optional, default: 10) - Maximum messages per batch
|
193
165
|
- `visibility_timeout:` (optional, default: 30) - Message visibility timeout in seconds
|
194
|
-
- `aws_access_key_id:` (optional) - AWS access key ID
|
195
|
-
- `aws_secret_access_key:` (optional) - AWS secret access key
|
196
|
-
- `aws_session_token:` (optional) - AWS session token for temporary credentials
|
197
|
-
- `aws_credentials:` (optional) - Pre-configured AWS credentials object
|
198
166
|
- `logger:` (optional) - Custom logger instance
|
199
167
|
|
200
168
|
### Environment Variables
|
@@ -11,20 +11,14 @@ module SQSProcessor
|
|
11
11
|
class Processor
|
12
12
|
attr_reader :sqs_client, :queue_url, :logger
|
13
13
|
|
14
|
-
def initialize(queue_url:,
|
15
|
-
|
16
|
-
aws_credentials: nil, logger: nil)
|
14
|
+
def initialize(queue_url:, aws_access_key_id:, aws_secret_access_key:, aws_region: 'us-east-1',
|
15
|
+
aws_session_token: nil, max_messages: 10, visibility_timeout: 30, logger: nil)
|
17
16
|
@queue_url = queue_url
|
18
|
-
@region = region
|
19
17
|
@max_messages = max_messages
|
20
18
|
@visibility_timeout = visibility_timeout
|
21
|
-
@aws_credentials = aws_credentials
|
22
|
-
@aws_access_key_id = aws_access_key_id
|
23
|
-
@aws_secret_access_key = aws_secret_access_key
|
24
|
-
@aws_session_token = aws_session_token
|
25
19
|
|
26
20
|
setup_logger(logger)
|
27
|
-
setup_sqs_client
|
21
|
+
setup_sqs_client(aws_access_key_id, aws_secret_access_key, aws_session_token, aws_region)
|
28
22
|
|
29
23
|
raise 'Queue URL is required.' unless @queue_url
|
30
24
|
end
|
@@ -37,19 +31,10 @@ module SQSProcessor
|
|
37
31
|
end
|
38
32
|
end
|
39
33
|
|
40
|
-
def setup_sqs_client
|
41
|
-
credentials =
|
42
|
-
@aws_credentials
|
43
|
-
elsif @aws_access_key_id && @aws_secret_access_key
|
44
|
-
Aws::Credentials.new(@aws_access_key_id, @aws_secret_access_key, @aws_session_token)
|
45
|
-
elsif ENV['DATA_SYNC_AWS_ACCESS_KEY_ID'] && ENV['DATA_SYNC_AWS_SECRET_ACCESS_KEY']
|
46
|
-
Aws::Credentials.new(ENV['DATA_SYNC_AWS_ACCESS_KEY_ID'], ENV['DATA_SYNC_AWS_SECRET_ACCESS_KEY'])
|
47
|
-
else
|
48
|
-
Aws::Credentials.new
|
49
|
-
end
|
50
|
-
|
34
|
+
def setup_sqs_client(aws_access_key_id, aws_secret_access_key, aws_session_token, aws_region)
|
35
|
+
credentials = Aws::Credentials.new(aws_access_key_id, aws_secret_access_key, aws_session_token)
|
51
36
|
@sqs_client = Aws::SQS::Client.new(
|
52
|
-
region:
|
37
|
+
region: aws_region,
|
53
38
|
credentials: credentials
|
54
39
|
)
|
55
40
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sqs_processor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Unni Tallman
|
@@ -125,14 +125,12 @@ description: A comprehensive Ruby gem for processing messages from Amazon SQS qu
|
|
125
125
|
with configurable message handling, error recovery, and extensible processing logic.
|
126
126
|
email:
|
127
127
|
- unni@bigbinary.com
|
128
|
-
executables:
|
129
|
-
- sqs_processor
|
128
|
+
executables: []
|
130
129
|
extensions: []
|
131
130
|
extra_rdoc_files: []
|
132
131
|
files:
|
133
132
|
- LICENSE
|
134
133
|
- README.md
|
135
|
-
- bin/sqs_processor
|
136
134
|
- lib/sqs_processor.rb
|
137
135
|
- lib/sqs_processor/processor.rb
|
138
136
|
- lib/sqs_processor/version.rb
|
data/bin/sqs_processor
DELETED
@@ -1,77 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require 'sqs_processor'
|
4
|
-
require 'optparse'
|
5
|
-
|
6
|
-
def parse_options
|
7
|
-
options = {
|
8
|
-
queue_url: nil,
|
9
|
-
region: nil,
|
10
|
-
max_messages: 10,
|
11
|
-
visibility_timeout: 30
|
12
|
-
}
|
13
|
-
|
14
|
-
OptionParser.new do |opts|
|
15
|
-
opts.banner = 'Usage: sqs_processor [options]'
|
16
|
-
|
17
|
-
opts.on('-q', '--queue-url URL', 'SQS Queue URL') do |url|
|
18
|
-
options[:queue_url] = url
|
19
|
-
end
|
20
|
-
|
21
|
-
opts.on('-r', '--region REGION', 'AWS Region') do |region|
|
22
|
-
options[:region] = region
|
23
|
-
end
|
24
|
-
|
25
|
-
opts.on('-m', '--max-messages NUMBER', Integer, 'Max messages per batch (default: 10)') do |num|
|
26
|
-
options[:max_messages] = num
|
27
|
-
end
|
28
|
-
|
29
|
-
opts.on('-v', '--visibility-timeout SECONDS', Integer, 'Visibility timeout in seconds (default: 30)') do |timeout|
|
30
|
-
options[:visibility_timeout] = timeout
|
31
|
-
end
|
32
|
-
|
33
|
-
opts.on('-h', '--help', 'Show this help message') do
|
34
|
-
puts opts
|
35
|
-
exit
|
36
|
-
end
|
37
|
-
end.parse!
|
38
|
-
|
39
|
-
options
|
40
|
-
end
|
41
|
-
|
42
|
-
if __FILE__ == $0
|
43
|
-
options = parse_options
|
44
|
-
|
45
|
-
begin
|
46
|
-
# Load environment variables
|
47
|
-
Dotenv.load
|
48
|
-
|
49
|
-
# Get queue URL from options or environment
|
50
|
-
queue_url = options[:queue_url] || ENV.fetch('DATA_SYNC_SQS_QUEUE_URL', nil)
|
51
|
-
|
52
|
-
unless queue_url
|
53
|
-
puts 'Error: Queue URL is required. Set DATA_SYNC_SQS_QUEUE_URL environment variable or use -q option.'
|
54
|
-
exit 1
|
55
|
-
end
|
56
|
-
|
57
|
-
processor = SQSProcessor::Processor.new(
|
58
|
-
queue_url: queue_url,
|
59
|
-
region: options[:region] || ENV['DATA_SYNC_AWS_REGION'] || 'us-east-1',
|
60
|
-
max_messages: options[:max_messages],
|
61
|
-
visibility_timeout: options[:visibility_timeout],
|
62
|
-
aws_access_key_id: ENV.fetch('DATA_SYNC_AWS_ACCESS_KEY_ID', nil),
|
63
|
-
aws_secret_access_key: ENV.fetch('DATA_SYNC_AWS_SECRET_ACCESS_KEY', nil),
|
64
|
-
aws_session_token: ENV.fetch('DATA_SYNC_AWS_SESSION_TOKEN', nil)
|
65
|
-
)
|
66
|
-
|
67
|
-
# Show queue attributes before starting
|
68
|
-
processor.get_queue_attributes
|
69
|
-
|
70
|
-
# Start processing messages
|
71
|
-
processor.process_messages
|
72
|
-
rescue StandardError => e
|
73
|
-
puts "Error: #{e.message}"
|
74
|
-
puts e.backtrace.join("\n")
|
75
|
-
exit 1
|
76
|
-
end
|
77
|
-
end
|