embulk-plugin-input-pcapng-files 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a831f929119e0209c353052fe5bf6e049dbafe36
+  data.tar.gz: 6c5d1a06ae6c8921c21a5577c03ad460eec672ea
+SHA512:
+  metadata.gz: c3a1b45cfd90162d1670a0f57ac2a78c10713feb26e9a44c09dffa72b6a2c1d615a253fda80ff4aaf082400daf6cea7a06d0e70d4d0ca69e85fd9aee7e868b85
+  data.tar.gz: fa8833f4b3d290e80ec1f9a1257890ce17edfb653e8ce60e1af956e4ca8798c382bc892b905643509f630bf4f542c3a342c5c9e59a15e8eb526534055b6e34d2

data/README.md

@@ -0,0 +1,11 @@
+embulk-plugin-input-pcapng-files
+================================
+
+Embulk plugin for pcapng files input.
+
+
+To extract specific field from pcapng files
+
+1. check sample\_config.yml, modify "paths" to where ever pcapng files are.
+2. specify fields to collect in "schema", name should correspond to tshark's field name
+

data/lib/embulk/input_pcapng_files.rb

@@ -0,0 +1,106 @@
+require "csv"
+
+module Embulk
+  class InputPcapngFiles < InputPlugin
+    # input plugin file name must be: embulk/input_<name>.rb
+    Plugin.register_input('pcapng_files', self)
+
+    def self.transaction(config, &control)
+      threads = config.param('threads', :integer, default: 2)
+      task = {
+        'paths' => [],
+        'done' => config.param('done', :array, default: []),
+        'paths_per_thread' => [],
+      }
+
+      task['paths'] = config.param('paths', :array, default: []).map {|path|
+        next [] unless Dir.exists?(path)
+        Dir.entries(path).sort.select {|entry| entry.match(/^.+\.pcapng$/)}.map {|entry|
+          path + "/" + entry
+        }
+      }.flatten
+      task['paths'] = task['paths'] - task['done']
+      task['paths_per_thread'] = task['paths'].each_slice(task['paths'].length / threads + 1).to_a
+
+      if task['paths'] == []
+        raise "no valid pcapng file found"
+      end
+
+      schema = config.param('schema', :array, default: [])
+      columns = []
+      columns << Column.new(0, "path", :string)
+      idx = 0
+      columns.concat schema.map{|c|
+        idx += 1
+        Column.new(idx, "#{c['name']}", c['type'].to_sym)
+      }
+
+      commit_reports = yield(task, columns, threads)
+      done = commit_reports.map{|hash| hash["done"]}.flatten.compact
+
+      return config.merge({ "done" => done })
+    end
+
+    def initialize(task, schema, index, page_builder)
+      super
+    end
+
+    attr_reader :task
+    attr_reader :schema
+    attr_reader :page_builder
+
+    def run
+      paths = task['paths_per_thread'][@index]
+      if paths == nil or paths == []
+        return {} # no task, no fail
+      end
+
+      paths.each do |path|
+        each_packet(path, schema[1..-1].map{|elm| elm.name}) do |hash|
+          entry = [ path ] + schema[1..-1].map {|c|
+            convert(hash[c.name], c.type)
+          }
+          @page_builder.add(entry)
+        end
+      end
+      @page_builder.finish # must call finish they say
+
+      return {"done" => paths}
+    end
+
+    private
+
+    def convert val, type
+      v = val
+      v = "" if val == nil
+      v = v.to_i if type == :long
+      return v
+    end
+
+    def build_options(fields)
+      options = ""
+      fields.each do |field|
+        options += "-e '#{field}' "
+      end
+      return options
+    end
+
+    def each_packet(path, fields, &block)
+      options = build_options(fields)
+      io = IO.popen("tshark -E separator=, #{options} -T fields -r #{path}")
+      while line = io.gets
+        array = [fields, CSV.parse(line).flatten].transpose
+        yield(Hash[*array.flatten])
+      end
+      io.close
+    end
+
+    def fetch_from_pcap(path, fields)
+      options = build_options(fields)
+      io = IO.popen("tshark -E separator=, #{options} -T fields -r #{path}")
+      data = io.read
+      io.close
+      return data
+    end
+  end
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: embulk-plugin-input-pcapng-files
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Naoya Kaneko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: embulk plugin for pcapng file input
+email: enukane@glenda9.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/embulk/input_pcapng_files.rb
+homepage: https://github.com/enukane/embulk-plugin-input-pcapng-files
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: embulk plugin for pcapng file input
+test_files: []