zim-ruby 1.0.1

data/lib/zim.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'stringio'
+require 'xz'
+
+require 'zim/file'
+require 'zim/structs'
+require 'zim/cluster'
+require 'zim/exceptions'
+require 'zim/url'
+
+# Zim module
+#
+# Example:
+#   zim = Zim::Zimfile.new('test.zim')
+#   p zim.urls.first.blob
+module Zim
+end
+

data/lib/zim/cluster.rb

@@ -0,0 +1,61 @@
+module Zim
+
+  # entry in the cluster list
+  class ClusterEntry
+
+    # compression of the cluster
+    attr_reader :compression
+
+    # read a cluster from the file
+    def initialize(f)
+      @compression = f.read_int8
+
+      @f = f
+      @pos = f.tell
+      @offsets = nil
+    end
+
+    # read the blob +blob+ from the cluster
+    def read_blob(blob)
+      case @compression
+      when 0, 1
+        read_blob_uncompressed(nil, blob)
+      when 4
+        read_blob_lzma(blob)
+      else
+        raise UnknownCompression
+      end
+    end
+
+    def read_blob_uncompressed(io, blob, seek = true) # :nodoc:
+      if @offsets.nil?
+        @offsets = Array.new
+
+        @f.seek(@pos, io) if seek
+
+        off = @f.read_int32(nil, io)
+        count = (off >> 2) - 1
+
+        @offsets << off
+        count.times do
+          @offsets << @f.read_int32(nil, io)
+        end
+      end
+
+      raise InvalidBlobNumber if (blob < 0) || (blob >= (@offsets.count - 1))
+
+      off = @offsets[blob]
+      next_off = @offsets[blob + 1]
+      len = next_off - off
+      off += @pos if seek
+
+      @f.read_str(len, off, io)
+    end
+
+    def read_blob_lzma(blob) # :nodoc:
+      @f.seek(@pos)
+      io = StringIO.new(XZ.decompress_stream(@f.file))
+      read_blob_uncompressed(io, blob, false)
+    end
+  end
+end

data/lib/zim/exceptions.rb

@@ -0,0 +1,15 @@
+module Zim
+
+  # file contains an invalid magic
+  class InvalidZimMagic < Exception
+  end
+
+  # the cluster compression is not supported
+  class UnknownCompression < Exception
+  end
+
+  # the blob number is out of range
+  class InvalidBlobNumber < Exception
+  end
+end
+

data/lib/zim/file.rb

@@ -0,0 +1,122 @@
+module Zim
+
+  # ZIM magic number
+  ZIM_MAGIC = 72173914
+
+  # Main class of the zim-ruby library
+  class ZimFile
+    # header informations (used internally)
+    attr_reader :header
+
+    # list of included mime types (used internally)
+    attr_reader :mime_types
+
+    # Directory of urls (used internally)
+    attr_reader :urls
+
+    # mapping of titles to urls (used internally)
+    attr_reader :titles
+
+    # cluster list (used internally)
+    attr_reader :clusters
+
+    # IO associated with this ZimFile
+    attr_reader :file
+
+    # load data from the given file
+    def initialize(filename)
+      @file = File.new(filename)
+
+      @header = FileHeader.new(self)
+      raise InvalidZimMagic if(@header.magic != ZIM_MAGIC)
+
+      seek(@header.mime_list_pos)
+      @mime_types = StringList.new(self)
+
+      seek(@header.url_pos)
+      @urls = Directory.new(self, @header.article_count, Url)
+
+      seek(@header.title_pos)
+      @titles = TitleList.new(self, @header.article_count)
+
+      seek(@header.cluster_pos)
+      @clusters = Directory.new(self, @header.cluster_count, ClusterEntry)
+    end
+
+    # access a url by full url (including namespace)
+    # e.g.: zim['/A/Table of Contents']
+    def [](idx)
+      @urls.detect { |x| x.to_s == idx }
+    end
+
+    def seek(pos, io = nil) # :nodoc:
+      io ||= @file
+      io.seek(pos)
+    end
+
+    def tell(io = nil) # :nodoc:
+      io ||= @file
+      io.tell
+    end
+
+    def read_int8(pos = nil, io = nil) # :nodoc:
+      io ||= @file
+      seek(pos, io) unless pos.nil?
+      io.read(1)[0]
+    end
+
+    def read_int16(pos = nil, io = nil) # :nodoc:
+      io ||= @file
+      seek(pos, io) unless pos.nil?
+      s = io.read(2)
+      (s[1] << 8) | s[0]
+    end
+
+    def read_int32(pos = nil, io = nil) # :nodoc:
+      io ||= @file
+      seek(pos, io) unless pos.nil?
+      s = io.read(4)
+      (s[3] << 24) |
+        (s[2] << 16) |
+        (s[1] << 8) |
+        s[0]
+    end
+
+    def read_int64(pos = nil, io = nil) # :nodoc:
+      io ||= @file
+      seek(pos, io) unless pos.nil?
+      s = io.read(8)
+      (s[7] << 56) |
+        (s[6] << 48) |
+        (s[5] << 40) |
+        (s[4] << 32) |
+        (s[3] << 24) |
+        (s[2] << 16) |
+        (s[1] << 8) |
+        s[0]
+    end
+
+    def read_str(size, pos = nil, io = nil) # :nodoc:
+      io ||= @file
+      seek(pos, io) unless pos.nil?
+
+      return '' if size == 0
+
+      io.read(size)
+    end
+
+    def read_cstr(pos = nil, io = nil) # :nodoc:
+      io ||= @file
+      seek(pos, io) unless pos.nil?
+
+      str = ''
+      begin
+        c = io.read(1)
+        str += c unless c[0] == 0
+      end while(c[0] != 0)
+
+      str
+    end
+  end
+end
+

data/lib/zim/structs.rb

@@ -0,0 +1,84 @@
+module Zim
+
+  # header data of a zim file
+  class FileHeader
+
+    # magic number
+    attr_reader :magic
+
+    # zim version
+    attr_reader :version
+
+    # uuid
+    attr_reader :uuid
+    attr_reader :article_count
+    attr_reader :cluster_count
+    attr_reader :url_pos # :nodoc:
+    attr_reader :title_pos # :nodoc:
+    attr_reader :cluster_pos # :nodoc:
+    attr_reader :mime_list_pos # :nodoc:
+
+    # main page index
+    attr_reader :main_page
+
+    # layout page index
+    attr_reader :layout_page
+    attr_reader :checksum_pos # :nodoc:
+
+    # read file header from a file
+    def initialize(f)
+      @magic         = f.read_int32
+      @version       = f.read_int32
+      @uuid          = f.read_str(16)
+      @article_count = f.read_int32
+      @cluster_count = f.read_int32
+      @url_pos       = f.read_int64
+      @title_pos     = f.read_int64
+      @cluster_pos   = f.read_int64
+      @mime_list_pos = f.read_int64
+      @main_page     = f.read_int32
+      @layout_page   = f.read_int32
+      @checksum_pos  = f.read_int64
+    end
+  end
+
+  class StringList < Array
+    def initialize(f)
+      super()
+
+      str = nil
+      begin
+        str = f.read_cstr
+        self << str unless str.length == 0
+      end while str.length > 0
+    end
+  end
+
+  class Directory < Array
+    def initialize(f, count, clazz)
+      super()
+
+      poses = Array.new
+      count.times do
+        pos = f.read_int64
+        poses << pos
+      end
+
+      poses.each do |pos|
+        f.seek(pos)
+        self << clazz.new(f)
+      end
+    end
+  end
+
+  class TitleList < Array
+    def initialize(f, count)
+      super()
+
+      count.times do
+        pos = f.read_int32
+        self << pos
+      end
+    end
+  end
+end

data/lib/zim/url.rb

@@ -0,0 +1,67 @@
+module Zim
+
+  # url entry class
+  class Url
+
+    attr_reader :mime_type
+    attr_reader :namespace
+    attr_reader :revision
+    attr_reader :redirect_index
+    attr_reader :cluster_number
+    attr_reader :blob_number
+    attr_reader :url
+    attr_reader :title
+    attr_reader :params
+
+    # read url from file
+    def initialize(f)
+      @f = f
+      mime_type = f.read_int16
+
+      if mime_type == 0xffff
+        # redirect
+        @mime_type      = nil
+        param_len       = f.read_int8
+        @namespace      = f.read_int8.chr
+        @revision       = f.read_int32
+        @redirect_index = f.read_int32
+        @cluster_number = nil
+        @blob_number    = nil
+        @url            = f.read_cstr
+        @title          = f.read_cstr
+        @params         = f.read_str(param_len)
+      else
+        @mime_type      = f.mime_types[mime_type]
+        param_len       = f.read_int8
+        @namespace      = f.read_int8.chr
+        @revision       = f.read_int32
+        @redirect_index = nil
+        @cluster_number = f.read_int32
+        @blob_number    = f.read_int32
+        @url            = f.read_cstr
+        @title          = f.read_cstr
+        @params         = f.read_str(param_len)
+      end
+    end
+
+    # checks if this url is a redirect
+    def redirect?
+      @mime_type.nil?
+    end
+
+    # receive the blob of this url
+    def blob
+      url = self
+      if redirect?
+        url = @f.urls[@redirect_index]
+      end
+
+      @f.clusters[@cluster_number].read_blob(@blob_number)
+    end
+
+    def to_s
+      "/#{@namespace}/#{@url}"
+    end
+  end
+end
+

data/zim-ruby.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name        = 'zim-ruby'
+  s.version     = '1.0.1'
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Christoph Plank']
+  s.email       = ['chrisistuff@gmail.com']
+  s.homepage    = 'http://rubygems.org/gems/zim-ruby'
+  s.summary     = %q{Library to read openzim (wikipedia) files}
+  s.description = %q{This library can be used to read openzim files like the ones exported by wikipedia}
+  s.has_rdoc    = false
+
+  #s.add_dependency('ruby-xz', '>= 0.0.2')
+  # added xz.rb directly to make it ruby 1.8 compatible
+
+  s.files         = `git ls-files`.split("\n")
+  s.require_paths = ['lib']
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification 
+name: zim-ruby
+version: !ruby/object:Gem::Version 
+  hash: 21
+  prerelease: 
+  segments: 
+  - 1
+  - 0
+  - 1
+  version: 1.0.1
+platform: ruby
+authors: 
+- Christoph Plank
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-11-18 00:00:00 Z
+dependencies: []
+
+description: This library can be used to read openzim files like the ones exported by wikipedia
+email: 
+- chrisistuff@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- LICENSE
+- README.markdown
+- examples/unpack.rb
+- lib/xz.rb
+- lib/zim.rb
+- lib/zim/cluster.rb
+- lib/zim/exceptions.rb
+- lib/zim/file.rb
+- lib/zim/structs.rb
+- lib/zim/url.rb
+- zim-ruby.gemspec
+homepage: http://rubygems.org/gems/zim-ruby
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Library to read openzim (wikipedia) files
+test_files: []
+