iudex-barc 1.0.0-java

data/History.rdoc

@@ -0,0 +1,2 @@
+=== 1.0.0 (2011-04-04)
+* Initial release.

data/Manifest.txt

@@ -0,0 +1,10 @@
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+pom.xml
+bin/iudex-barc
+bin/iudex-http-record
+lib/iudex-barc/base.rb
+lib/iudex-barc.rb
+lib/iudex-barc/iudex-barc-1.0.0.jar

data/README.rdoc

@@ -0,0 +1,25 @@
+= iudex-barc
+
+* http://github.com/dekellum/iudex
+
+== Description
+
+Iudex is a general purpose web crawler and feed processor in
+ruby/java. The iudex-barc gem contains support for the BARC Basic
+ARChive format.
+
+== License
+
+Copyright (c) 2008-2011 David Kellum
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/Rakefile

@@ -0,0 +1,40 @@
+# -*- ruby -*-
+
+$LOAD_PATH << './lib'
+require 'iudex-barc/base'
+
+require 'rubygems'
+gem     'rjack-tarpit', '~> 1.2'
+require 'rjack-tarpit'
+
+t = RJack::TarPit.new( 'iudex-barc',
+                       Iudex::BARC::VERSION,
+                       :no_assembly, :java_platform )
+
+t.specify do |h|
+  h.developer( "David Kellum", "dek-oss@gravitext.com" )
+  h.extra_deps += [ [ 'rjack-slf4j',    '~> 1.6.1'  ],
+                    [ 'gravitext-util', '~> 1.5.0'  ],
+                    [ 'iudex-http',     '~> 1.0.0'  ] ]
+
+  h.testlib = :minitest
+  h.extra_dev_deps << [ 'minitest',     '>= 1.7.1', '< 2.1' ]
+end
+
+file 'Manifest.txt' => "lib/#{t.name}/base.rb"
+
+task :check_pom_version do
+  t.test_line_match( 'pom.xml', /<version>/, /#{t.version}/ )
+end
+task :check_history_version do
+  t.test_line_match( 'History.rdoc', /^==/, / #{t.version} / )
+end
+task :check_history_date do
+  t.test_line_match( 'History.rdoc', /^==/, /\([0-9\-]+\)$/ )
+end
+
+task :gem  => [ :check_pom_version, :check_history_version                      ]
+task :tag  => [ :check_pom_version, :check_history_version, :check_history_date ]
+task :push => [                                             :check_history_date ]
+
+t.define_tasks

data/bin/iudex-barc

@@ -0,0 +1,176 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+
+require 'optparse'
+
+module IudexBinScript
+
+  require 'rjack-logback'
+  include RJack
+
+  Logback.config_console( :level => Logback::INFO, :stderr => true )
+
+  require 'iudex-barc'
+
+  class BARCTool
+    include Iudex
+    import 'com.gravitext.util.Streams'
+
+    def initialize
+      @offset = 0
+      @sections = ALL_SECTIONS
+      @show_replaced = false
+    end
+
+    COMMANDS = [ :show ]
+    ALL_SECTIONS = [ :meta, :request, :response, :body ]
+
+    HEADER_TITLES = { :meta => "META",
+                      :request => "RQST",
+                      :response => "RESP" }
+
+    def run( args = ARGV )
+      command = parse_args( args )
+      self.send( command, args )
+    end
+
+    def parse_args( args = ARGV )
+      osecs = []
+
+      parser = OptionParser.new do |opts|
+        opts.banner = ( "Usage: iudex-barc [options] {show} BARCFile...\n" +
+                        "Options:\n" )
+
+        opts.on( "-v", "--version", "Display version and exit" ) do |file|
+          puts "iudex-barc: #{BARC::VERSION}"
+          exit 1
+        end
+
+        opts.on( "-o", "--offset N", String,
+                 "Offset into (first) BARCFile (one record)" ) do |offset|
+          @offset = Integer( offset )
+        end
+
+        { :meta => 'm', :request => 'q', :response => 'r',
+          :body => 'b' }.each do |sec,c|
+          opts.on( '-' + c,
+                   '--' + sec.to_s,
+                   "Show #{sec} #{ 'headers' unless sec == :body }" ) do
+            osecs << sec
+          end
+        end
+
+        opts.on( "-x", "--show-replaced",
+                 "Show replaced records as well." ) do
+          @show_replaced = true
+        end
+
+        opts.on_tail( "-h", "--help", "Show help and exit" ) do
+          opts.usage
+        end
+      end
+
+      def parser.usage
+        puts self
+        puts
+        puts( "Commands:")
+        puts( "    show: Dump BARC record details." )
+        exit 1
+      end
+
+      parser.parse!( args )
+
+      @sections = osecs unless osecs.empty?
+
+      bsec, @h_sections = @sections.partition { |s| s == :body }
+      @body_section = ! bsec.empty?
+
+      command = args.shift
+
+      parser.usage unless command
+      command = command.to_sym
+
+      parser.usage unless COMMANDS.include?( command )
+      command
+    end
+
+    def show( barcs )
+      if @offset != 0
+        bfile = barc_open( barcs.first )
+        display( bfile.read( @offset ) )
+      else
+        barcs.each do |bname|
+          bfile = barc_open( bname )
+          rr = bfile.reader
+          while( rec = rr.next )
+            if @show_replaced || rec.type.chr != 'R'
+              display( rec, ( bname if barcs.length > 1 ) )
+            end
+          end
+        end
+      end
+    end
+
+    def display( rec, meta = nil )
+      puts( "-BARC1 %c%s : 0x%x %s" %
+            [ rec.type,
+              rec.compressed? ? 'C':'P',
+              rec.offset,
+              meta ] )
+
+      @h_sections.each do |ht|
+        display_headers( ht, rec.send( "#{ht}_headers" ) )
+      end
+
+      dump_body( rec ) if @body_section
+    end
+
+    def display_headers( htype, headers )
+      unless headers.empty?
+        puts "=#{ HEADER_TITLES[ htype ] }="
+        headers.each { |h| puts h }
+        puts
+      end
+    end
+
+    def dump_body( rec )
+      body_in = rec.body_input_stream
+      if body_in && ( c = body_in.read ) != -1
+        out = Java::java.lang.System::out
+        puts "=BODY="
+        out.write( c )
+        Streams::copy( body_in, out )
+        puts
+      end
+    end
+
+    def barc_open( bname )
+      #FIXME: Avoid opening new barc if not existing
+      raise "BARC File '#{bname}' not found" unless File.exist?( bname )
+      BARC::BARCFile.new( Java::java.io.File.new( bname ) )
+    end
+  end
+
+  bt = BARCTool.new
+  bt.run
+
+end

data/bin/iudex-http-record

@@ -0,0 +1,51 @@
+#!/usr/bin/env jruby
+# -*- ruby -*-
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.join( File.dirname(__FILE__), "..", "lib" )
+
+require 'rubygems'
+require 'rjack-logback'
+RJack::Logback.config_console
+
+#RJack::Logback[ "org.apache.commons.httpclient" ].level = RJack::Logback::INFO
+#RJack::Logback[ "iudex" ].level = RJack::Logback::DEBUG
+
+require 'iudex-barc'
+require 'iudex-httpclient-3' #FIXME: Unspecified dependency
+
+import 'iudex.httpclient3.HTTPClient3'
+import 'iudex.barc.http.BARCResponseHandler'
+import 'iudex.barc.BARCFile'
+
+hmanager = RJack::HTTPClient3::ManagerFacade.new
+hmanager.start
+
+hclient = HTTPClient3.new( hmanager.client )
+
+barc_file = BARCFile.new( java.io.File.new( './record.barc' ) ) #FIXME: param
+barc_file.truncate #FIXME: Optional
+
+handler = BARCResponseHandler.new( barc_file )
+handler.do_compress = false #FIXME: Option
+
+hsession = hclient.createSession;
+hsession.url = 'http://gravitext.com/blog' #FIXME: param
+
+hclient.request( hsession, handler )
+
+hmanager.shutdown

data/lib/iudex-barc/base.rb

@@ -0,0 +1,23 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module Iudex
+  module BARC
+    VERSION = '1.0.0'
+
+    LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
+  end
+end

data/lib/iudex-barc.rb

@@ -0,0 +1,33 @@
+#--
+# Copyright (c) 2008-2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'rjack-slf4j'
+require 'gravitext-util'
+
+require 'iudex-http'
+
+require 'iudex-barc/base'
+
+require 'java'
+
+module Iudex
+  module BARC
+    require "#{LIB_DIR}/iudex-barc-#{VERSION}.jar"
+
+    import 'iudex.barc.BARCDirectory'
+    import 'iudex.barc.BARCFile'
+  end
+end

data/pom.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>iudex</groupId>
+  <artifactId>iudex-barc</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0.0</version>
+  <name>Iudex Basic ARChive Format</name>
+
+  <parent>
+    <groupId>iudex</groupId>
+    <artifactId>iudex-parent</artifactId>
+    <version>1.0</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <dependencies>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.gravitext</groupId>
+      <artifactId>gravitext-util</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>iudex</groupId>
+      <artifactId>iudex-http</artifactId>
+      <version>[1.0,1.1)</version>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+      <plugin>
+        <!-- Parent settings -->
+        <artifactId>maven-source-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification 
+name: iudex-barc
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 1.0.0
+platform: java
+authors: 
+  - David Kellum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-04 00:00:00 -07:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: rjack-slf4j
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.6.1
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: gravitext-util
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.5.0
+    type: :runtime
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: iudex-http
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.0.0
+    type: :runtime
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: 1.7.1
+        - - <
+          - !ruby/object:Gem::Version 
+            version: "2.1"
+    type: :development
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: rjack-tarpit
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 1.3.0
+    type: :development
+    version_requirements: *id005
+description: |-
+  Iudex is a general purpose web crawler and feed processor in
+  ruby/java. The iudex-barc gem contains support for the BARC Basic
+  ARChive format.
+email: 
+  - dek-oss@gravitext.com
+executables: 
+  - iudex-barc
+  - iudex-http-record
+extensions: []
+
+extra_rdoc_files: 
+  - Manifest.txt
+  - History.rdoc
+  - README.rdoc
+files: 
+  - History.rdoc
+  - Manifest.txt
+  - README.rdoc
+  - Rakefile
+  - pom.xml
+  - bin/iudex-barc
+  - bin/iudex-http-record
+  - lib/iudex-barc/base.rb
+  - lib/iudex-barc.rb
+  - lib/iudex-barc/iudex-barc-1.0.0.jar
+has_rdoc: true
+homepage: http://github.com/dekellum/iudex
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.rdoc
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: iudex-barc
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: Iudex is a general purpose web crawler and feed processor in ruby/java
+test_files: []
+