wonderdog 0.0.1

data/test/foo.json

@@ -0,0 +1,3 @@
+{"character":"a","value":"1"}
+{"character":"b","value":"2"}
+{"character":"c","value":"3"}

data/test/foo.tsv

@@ -0,0 +1,3 @@
+a	1
+b	2
+c	3

data/test/test_dump.pig

@@ -0,0 +1,19 @@
+--
+-- This tests loading data from elasticsearch
+--
+
+%default ES_JAR_DIR '/usr/local/Cellar/elasticsearch/0.18.7/libexec'
+%default ES_YAML    '/usr/local/Cellar/elasticsearch/0.18.7/config/elasticsearch.yml'
+%default PLUGINS    '/usr/local/Cellar/elasticsearch/0.18.7/plugins'
+
+%default INDEX      'foo_test'
+%default OBJ        'foo'        
+
+register $ES_JAR_DIR/*.jar;
+register target/wonderdog*.jar;
+
+--
+-- Will load the data as (doc_id, contents) tuples where the contents is the original json source from elasticsearch
+--
+foo = LOAD 'es://$INDEX/$OBJ' USING com.infochimps.elasticsearch.pig.ElasticSearchStorage('$ES_YAML', '$PLUGINS') AS (doc_id:chararray, contents:chararray);
+DUMP foo;

data/test/test_json_loader.pig

@@ -0,0 +1,21 @@
+--
+-- This tests the json indexer. Run in local mode with 'pig -x local test/test_json_loader.pig'
+--
+	
+%default ES_JAR_DIR '/usr/local/Cellar/elasticsearch/0.18.7/libexec'
+%default ES_YAML    '/usr/local/Cellar/elasticsearch/0.18.7/config/elasticsearch.yml'
+%default PLUGINS    '/usr/local/Cellar/elasticsearch/0.18.7/plugins'
+
+%default INDEX      'foo_test'
+%default OBJ        'foo'        
+
+register $ES_JAR_DIR/*.jar;
+register target/wonderdog*.jar;
+
+foo = LOAD 'test/foo.json' AS (data:chararray);
+
+--
+-- Query parameters let elasticsearch output format that we're storing json data and
+-- want to use a bulk request size of 1 record.
+--
+STORE foo INTO 'es://$INDEX/$OBJ?json=true&size=1' USING com.infochimps.elasticsearch.pig.ElasticSearchStorage('$ES_YAML', '$PLUGINS');

data/test/test_tsv_loader.pig

@@ -0,0 +1,16 @@
+--
+-- This tests the tsv indexer. Run in local mode with 'pig -x local test/test_tsv_loader.pig'
+--
+%default ES_JAR_DIR '/usr/local/Cellar/elasticsearch/0.18.7/libexec'
+%default ES_YAML    '/usr/local/Cellar/elasticsearch/0.18.7/config/elasticsearch.yml'
+%default PLUGINS    '/usr/local/Cellar/elasticsearch/0.18.7/plugins'
+
+%default INDEX      'foo_test'
+%default OBJ        'foo'        
+
+register $ES_JAR_DIR/*.jar;
+register target/wonderdog*.jar;
+
+foo = LOAD 'test/foo.tsv' AS (character:chararray, value:int);
+
+STORE foo INTO 'es://$INDEX/$OBJ?json=false&size=1' USING com.infochimps.elasticsearch.pig.ElasticSearchStorage('$ES_YAML', '$PLUGINS');

data/wonderdog.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/wonderdog/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'wonderdog'
+  gem.homepage    = 'https://github.com/infochimps-labs/wonderdog'
+  gem.licenses    = ["Apache 2.0"]
+  gem.email       = 'coders@infochimps.com'
+  gem.authors     = ['Infochimps', 'Philip (flip) Kromer', 'Jacob Perkins', 'Travis Dempsey', 'Dhruv Bansal']
+  gem.version     = Wonderdog::VERSION
+
+  gem.summary     = 'Make Hadoop and ElasticSearch play together nicely.'
+  gem.description = <<-EOF
+  Wonderdog provides code in both Ruby and Java to make Elasticsearch
+  a more fully-fledged member of both the Hadoop and Wukong
+  ecosystems.
+
+  For the Java side, Wonderdog provides InputFormat and OutputFormat
+  classes for use with Hadoop (esp. Hadoop Streaming) and Pig.
+
+  For the Ruby side, Wonderdog provides extensions for wu-hadoop to
+  make running Hadoop Streaming jobs written in Wukong against
+  ElasticSearch easier.
+EOF
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.executables   = []
+  gem.test_files    = gem.files.grep(/^spec/)
+  gem.require_paths = ['lib']
+
+  gem.add_dependency('wukong', '3.0.0.pre2')
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: wonderdog
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Infochimps
+- Philip (flip) Kromer
+- Jacob Perkins
+- Travis Dempsey
+- Dhruv Bansal
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: wukong
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.0.pre2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.0.pre2
+description: ! "  Wonderdog provides code in both Ruby and Java to make Elasticsearch\n
+  \ a more fully-fledged member of both the Hadoop and Wukong\n  ecosystems.\n\n  For
+  the Java side, Wonderdog provides InputFormat and OutputFormat\n  classes for use
+  with Hadoop (esp. Hadoop Streaming) and Pig.\n\n  For the Ruby side, Wonderdog provides
+  extensions for wu-hadoop to\n  make running Hadoop Streaming jobs written in Wukong
+  against\n  ElasticSearch easier.\n"
+email: coders@infochimps.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- CHANGELOG.md
+- LICENSE.md
+- README.md
+- Rakefile
+- bin/estool
+- bin/estrus.rb
+- bin/wonderdog
+- config/elasticsearch-example.yml
+- config/elasticsearch.in.sh
+- config/logging.yml
+- config/more_settings.yml
+- config/run_elasticsearch-2.sh
+- config/ufo_config.json
+- lib/wonderdog.rb
+- lib/wonderdog/configuration.rb
+- lib/wonderdog/hadoop_invocation_override.rb
+- lib/wonderdog/index_and_mapping.rb
+- lib/wonderdog/timestamp.rb
+- lib/wonderdog/version.rb
+- notes/README-benchmarking.txt
+- notes/README-read_tuning.textile
+- notes/benchmarking-201011.numbers
+- notes/cluster_notes.md
+- notes/notes.txt
+- notes/pigstorefunc.pig
+- pom.xml
+- spec/spec_helper.rb
+- spec/support/driver_helper.rb
+- spec/support/integration_helper.rb
+- spec/wonderdog/hadoop_invocation_override_spec.rb
+- spec/wonderdog/index_and_type_spec.rb
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchInputFormat.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchOutputCommitter.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchOutputFormat.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchSplit.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchStreamingInputFormat.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchStreamingOutputCommitter.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchStreamingOutputFormat.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchStreamingRecordReader.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchStreamingRecordWriter.java
+- src/main/java/com/infochimps/elasticsearch/ElasticSearchStreamingSplit.java
+- src/main/java/com/infochimps/elasticsearch/ElasticTest.java
+- src/main/java/com/infochimps/elasticsearch/hadoop/util/HadoopUtils.java
+- src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchIndex.java
+- src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchJsonIndex.java
+- src/main/java/com/infochimps/elasticsearch/pig/ElasticSearchStorage.java
+- test/foo.json
+- test/foo.tsv
+- test/test_dump.pig
+- test/test_json_loader.pig
+- test/test_tsv_loader.pig
+- wonderdog.gemspec
+homepage: https://github.com/infochimps-labs/wonderdog
+licenses:
+- Apache 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Make Hadoop and ElasticSearch play together nicely.
+test_files:
+- spec/spec_helper.rb
+- spec/support/driver_helper.rb
+- spec/support/integration_helper.rb
+- spec/wonderdog/hadoop_invocation_override_spec.rb
+- spec/wonderdog/index_and_type_spec.rb
+has_rdoc: