logstash-input-azure_storage_blob 1.0.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b820217f7a001f86131e31c24d98c78e96a470593318f90d5fd180470bdf4b68
+  data.tar.gz: 94db9edbbf831573d893e54a18441a985af0de7d8097737cbb1c104419639764
+SHA512:
+  metadata.gz: 88f88a7545f9c4763d6fa566bdf1aa7408fe2cdfa44aabc65142e935eb83835ccfdab2dc2e68a00995c135932fee638869b65b6fc5dbf6aca50f53f71b0c9351
+  data.tar.gz: 48b06fab2c17c6a741c6bbfc20327d0c129a5f22499b32b9e7feac64753fe8f34f85535b9503748ce6c8c1b6c4d10cbb3499dacd0437dda5d54b57c5e9a9890a

data/Gemfile

@@ -0,0 +1,12 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/README.md

@@ -0,0 +1,301 @@
+# Logstash Input Plugin for Azure Blob Storage
+
+A [Logstash](https://www.elastic.co/logstash) input plugin that reads blobs from [Azure Blob Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/) and streams their content line-by-line into your Logstash pipeline.
+
+Built with Microsoft's first-party Java Azure SDKs, this plugin runs natively on the JVM alongside Logstash - no Ruby shims or REST wrappers required. It supports Azure Commercial, Azure Government, and Azure China clouds out of the box.
+
+## Features
+
+- **Streaming architecture** - processes blobs line-by-line without loading entire files into memory
+- **Multi-replica safe** - blob leases prevent duplicate processing across multiple Logstash instances
+- **Three state-tracking strategies** - choose the right trade-off between permissions, portability, and coordination
+- **Azure Government and sovereign clouds** - first-class support with automatic cloud detection
+- **Flexible authentication** - DefaultAzureCredential (managed identity, workload identity, CLI), connection strings, or shared keys
+
+## Installation
+
+### From RubyGems
+
+```bash
+/usr/share/logstash/bin/logstash-plugin install logstash-input-azure_storage_blob
+```
+
+### From GitHub Release
+
+Download the `.gem` file from [Releases](https://github.com/kfriede/logstash-input-azure_storage_blob/releases), then install it:
+
+```bash
+/usr/share/logstash/bin/logstash-plugin install --no-verify logstash-input-azure_storage_blob-1.0.0-java.gem
+```
+
+### From Source
+
+```bash
+git clone https://github.com/kfriede/logstash-input-azure_storage_blob.git
+cd logstash-input-azure_storage_blob
+./gradlew gem
+/usr/share/logstash/bin/logstash-plugin install --no-verify logstash-input-azure_storage_blob-1.0.0-java.gem
+```
+
+### Verify
+
+```bash
+/usr/share/logstash/bin/logstash-plugin list | grep azure_storage_blob
+```
+
+## Quick Start
+
+The simplest configuration requires only a storage account and container name. Authentication defaults to `DefaultAzureCredential`, which automatically picks up managed identity, workload identity, Azure CLI credentials, or environment variables.
+
+```ruby
+input {
+  azure_storage_blob {
+    storage_account => "mystorageaccount"
+    container       => "logs"
+  }
+}
+
+output {
+  stdout { codec => rubydebug }
+}
+```
+
+## Configuration Reference
+
+### Required Settings
+
+| Setting | Type | Description |
+|---------|------|-------------|
+| `storage_account` | string | Azure Storage account name. |
+| `container` | string | Name of the blob container to read from. |
+
+### Authentication
+
+| Setting | Type | Default | Description |
+|---------|------|---------|-------------|
+| `auth_method` | string | `"default"` | Authentication method. One of `default`, `connection_string`, or `shared_key`. |
+| `connection_string` | string | `""` | Azure Storage connection string. Required when `auth_method => "connection_string"`. |
+| `storage_key` | string | `""` | Storage account access key. Required when `auth_method => "shared_key"`. |
+
+**`auth_method` options:**
+
+| Value | Description | When to use |
+|-------|-------------|-------------|
+| `default` | Uses `DefaultAzureCredential` - automatically tries managed identity, workload identity, environment variables, and Azure CLI in order. | Production deployments on Azure (AKS, VMs, App Service). |
+| `connection_string` | Authenticates with a full connection string. | Quick setup, development, or environments without Azure AD. |
+| `shared_key` | Authenticates with the storage account access key. | When you need key-based auth without a full connection string. |
+
+### Azure Environment
+
+| Setting | Type | Default | Description |
+|---------|------|---------|-------------|
+| `cloud` | string | `""` (auto-detect) | Azure cloud environment. One of `AzureCloud`, `AzureUSGovernment`, or `AzureChinaCloud`. When empty, auto-detects from the `AZURE_AUTHORITY_HOST` environment variable. |
+| `blob_endpoint` | string | `""` | Explicit blob service endpoint URL. Overrides the endpoint derived from `cloud` and `storage_account`. Use for private endpoints or the Azurite emulator. |
+
+**Cloud endpoints:**
+
+| `cloud` value | Storage endpoint | Auth endpoint |
+|---------------|-----------------|---------------|
+| `AzureCloud` | `*.blob.core.windows.net` | `login.microsoftonline.com` |
+| `AzureUSGovernment` | `*.blob.core.usgovcloudapi.net` | `login.microsoftonline.us` |
+| `AzureChinaCloud` | `*.blob.core.chinacloudapi.cn` | `login.chinacloudapi.cn` |
+
+### State Tracking
+
+| Setting | Type | Default | Description |
+|---------|------|---------|-------------|
+| `tracking_strategy` | string | `"tags"` | How the plugin tracks which blobs have been processed. One of `tags`, `container`, or `registry`. |
+| `archive_container` | string | `"archive"` | Container where completed blobs are moved. Only used with `container` strategy. |
+| `error_container` | string | `"errors"` | Container where failed blobs are moved. Only used with `container` strategy. |
+| `registry_path` | string | `"/data/registry.db"` | Path to the local SQLite database file. Only used with `registry` strategy. |
+
+**`tracking_strategy` options:**
+
+| Strategy | How it works | RBAC required | Multi-replica safe |
+|----------|-------------|---------------|-------------------|
+| `tags` | Writes blob index tags (`logstash_status`, etc.) to mark blobs as processing/completed/failed. Uses blob leases for coordination. | Storage Blob Data Owner | Yes |
+| `container` | Moves completed blobs to an archive container and failed blobs to an error container. Uses blob leases for coordination. | Storage Blob Data Contributor | Yes |
+| `registry` | Tracks state in a local SQLite database. No Azure-side state changes. | Storage Blob Data Reader | **No** - single instance only |
+
+### Polling
+
+| Setting | Type | Default | Description |
+|---------|------|---------|-------------|
+| `poll_interval` | number | `30` | Seconds to wait between poll cycles. |
+| `prefix` | string | `""` | Only process blobs whose name starts with this prefix. Empty means all blobs. |
+| `blob_batch_size` | number | `10` | Maximum number of blobs to process per poll cycle. |
+
+### Processing
+
+| Setting | Type | Default | Description |
+|---------|------|---------|-------------|
+| `skip_empty_lines` | boolean | `true` | Whether to skip empty lines when reading blob content. |
+
+### Lease Coordination
+
+These settings apply only to the `tags` and `container` tracking strategies.
+
+| Setting | Type | Default | Description |
+|---------|------|---------|-------------|
+| `lease_duration` | number | `30` | Blob lease duration in seconds. Must be between 15 and 60 (Azure requirement). |
+| `lease_renewal` | number | `20` | How often to renew the lease, in seconds. Should be less than `lease_duration`. |
+
+## Event Metadata
+
+Every event produced by this plugin includes the following `@metadata` fields:
+
+| Field | Description |
+|-------|-------------|
+| `[@metadata][azure_blob_name]` | Name of the source blob. |
+| `[@metadata][azure_blob_container]` | Name of the source container. |
+| `[@metadata][azure_blob_storage_account]` | Storage account the blob was read from. |
+| `[@metadata][azure_blob_line_number]` | Line number within the blob (1-based). |
+| `[@metadata][azure_blob_last_modified]` | Last-modified timestamp of the blob (ISO 8601). |
+
+You can use these in filters and outputs:
+
+```ruby
+filter {
+  mutate {
+    add_field => {
+      "source_blob" => "%{[@metadata][azure_blob_name]}"
+    }
+  }
+}
+```
+
+## Examples
+
+### Ingest JSON logs from Azure Government
+
+```ruby
+input {
+  azure_storage_blob {
+    storage_account   => "govlogsstorage"
+    container         => "application-logs"
+    cloud             => "AzureUSGovernment"
+    tracking_strategy => "tags"
+    prefix            => "prod/"
+    poll_interval     => 10
+  }
+}
+
+filter {
+  json { source => "message" }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    index => "app-logs-%{+YYYY.MM.dd}"
+  }
+}
+```
+
+### Connection string auth with container-based tracking
+
+```ruby
+input {
+  azure_storage_blob {
+    storage_account    => "myaccount"
+    container          => "incoming"
+    auth_method        => "connection_string"
+    connection_string  => "${AZURE_STORAGE_CONNECTION_STRING}"
+    tracking_strategy  => "container"
+    archive_container  => "processed"
+    error_container    => "failed"
+  }
+}
+
+output {
+  stdout { codec => rubydebug }
+}
+```
+
+Blobs are moved from `incoming` to `processed` on success, or to `failed` on error. Create all three containers before starting the pipeline.
+
+### Lightweight single-instance setup with SQLite registry
+
+```ruby
+input {
+  azure_storage_blob {
+    storage_account   => "myaccount"
+    container         => "data"
+    auth_method       => "shared_key"
+    storage_key       => "${AZURE_STORAGE_KEY}"
+    tracking_strategy => "registry"
+    registry_path     => "/var/lib/logstash/blob-registry.db"
+    blob_batch_size   => 50
+    poll_interval     => 60
+  }
+}
+
+output {
+  file {
+    path => "/var/log/ingested/%{[@metadata][azure_blob_name]}.log"
+  }
+}
+```
+
+The `registry` strategy only requires `Storage Blob Data Reader` permissions and stores state locally. Do not run multiple Logstash instances with this strategy - they will process the same blobs independently.
+
+### Multi-replica deployment on Kubernetes
+
+```ruby
+input {
+  azure_storage_blob {
+    storage_account   => "prodlogs"
+    container         => "events"
+    tracking_strategy => "tags"
+    poll_interval     => 5
+    blob_batch_size   => 20
+    lease_duration    => 60
+    lease_renewal     => 40
+  }
+}
+```
+
+With the `tags` or `container` strategy, you can safely run multiple Logstash replicas against the same container. Blob leases ensure each blob is processed by exactly one replica. On AKS with workload identity, `DefaultAzureCredential` picks up the pod identity automatically and the `cloud` setting is auto-detected from `AZURE_AUTHORITY_HOST`.
+
+### Route events by blob name
+
+```ruby
+input {
+  azure_storage_blob {
+    storage_account => "myaccount"
+    container       => "logs"
+  }
+}
+
+output {
+  if [@metadata][azure_blob_name] =~ /^access-logs/ {
+    elasticsearch {
+      index => "access-logs-%{+YYYY.MM.dd}"
+    }
+  } else if [@metadata][azure_blob_name] =~ /^error-logs/ {
+    elasticsearch {
+      index => "error-logs-%{+YYYY.MM.dd}"
+    }
+  }
+}
+```
+
+## Building from Source
+
+Requires Java 11+ and a local Logstash installation.
+
+```bash
+# Run unit tests
+./gradlew test
+
+# Run integration tests (requires Azurite)
+docker run -d --name azurite -p 10000:10000 mcr.microsoft.com/azure-storage/azurite
+./gradlew integrationTest
+
+# Build the gem
+./gradlew gem
+```
+
+## License
+
+Apache-2.0

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/logstash/inputs/azure_storage_blob.rb

@@ -0,0 +1,12 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash-input-azure_storage_blob_jars"
+require "java"
+
+class LogStash::Inputs::AzureStorageBlob < LogStash::Inputs::Base
+  config_name "azure_storage_blob"
+
+  def self.javaClass() Java::com.azure.logstash.input.AzureStorageBlob.java_class; end
+end

data/lib/logstash-input-azure_storage_blob_jars.rb

@@ -0,0 +1,5 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
+# encoding: utf-8
+
+require 'jar_dependencies'
+require_jar('com.azure.logstash.input', 'logstash-input-azure_storage_blob', '1.0.0')

data/logstash-input-azure_storage_blob.gemspec

@@ -0,0 +1,23 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN.
+Gem::Specification.new do |s|
+  s.name            = 'logstash-input-azure_storage_blob'
+  s.version         = ::File.read('VERSION').split('\n').first
+  s.licenses        = ['Apache-2.0']
+  s.summary         = 'Logstash input plugin for Azure Blob Storage'
+  s.description     = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
+  s.authors         = ['Microsoft']
+  s.email           = ['']
+  s.homepage        = 'https://github.com/Azure/logstash-input-azure_storage_blob'
+  s.platform        = 'java'
+  s.require_paths   = ['lib', 'vendor/jar-dependencies']
+
+  s.files = Dir["lib/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input', 'java_plugin' => 'true'}
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'jar-dependencies'
+  s.add_development_dependency 'logstash-devutils'
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-azure_storage_blob
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: java
+authors:
+- Microsoft
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-02-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: jar-dependencies
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email:
+- ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- VERSION
+- lib/logstash-input-azure_storage_blob_jars.rb
+- lib/logstash/inputs/azure_storage_blob.rb
+- logstash-input-azure_storage_blob.gemspec
+- vendor/jar-dependencies/com/azure/logstash/input/logstash-input-azure_storage_blob/1.0.0/logstash-input-azure_storage_blob-1.0.0.jar
+homepage: https://github.com/Azure/logstash-input-azure_storage_blob
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+  java_plugin: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- vendor/jar-dependencies
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Logstash input plugin for Azure Blob Storage
+test_files: []