mmapper 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 624fb225f474ed8da83938adf19dd4e1a3727c8c4c0660d316f6896f620ca5d1
+  data.tar.gz: 97607d8736e460e4f788efde73cd35652b5abe850a8dfcc0accfaf68ab29ede1
+SHA512:
+  metadata.gz: a91ef5d57628aa47c30b5b0dd4957d41373440b7691792baa93647e57b279bcb2d466ce63f24f43285b6f93e0c711fcac882e6b54e2ebf8ddd5c043a8604a3d7
+  data.tar.gz: a5e0e6825d701945192923dadabbe87984a04aa2a3e74031a4e5d183abfd0f43619e531e585422416fea6711e4535e562be813b657f1fc0017fd82a0808a6726

data/README.md

@@ -0,0 +1,60 @@
+# Mmapper
+
+## Overview
+Mmapper is a Ruby gem that provides a high-performance interface for **memory-mapping (mmap) files** using a Go extension. This allows efficient read access to large files without loading them into memory. As a proof of concept, a binary search function has been implemented to quickly find lines that match a given prefix.
+
+For testing purposes, the ICANN .com zone file (22.54gb, over 400 million lines) was loaded with Mmapper and the average time to perform a binary search for a given string was 4.56ms (n=1000), see `benchmark_mmapper.rb`.
+
+## Features
+- **Memory-maps large files** for fast, low-memory access.
+- **Supports multiple mmap instances** at the same time.
+- **Uses Ruby FFI** to interact with a Go extension.
+- **Binary search** for finding lines that start with a given prefix.
+
+## Installation
+You can install the gem locally after building it:
+
+```sh
+gem install mmapper
+```
+
+## Usage
+
+### Loading a File
+To mmap a file, use `Mmapper.load_file`, which returns an instance:
+
+```ruby
+require "mmapper"
+
+mmap = Mmapper.load_file("/path/to/file.txt")
+```
+
+### Searching for a Matching Line
+The `find_matching_line` method performs a **binary search** for a line that starts with the given prefix:
+
+```ruby
+result = mmap.find_matching_line("example")
+puts result.nil? ? "Not found" : "Found: #{result}"
+```
+
+### Working with Multiple Files
+Each mmap instance is independent, allowing you to search different files simultaneously:
+
+```ruby
+m1 = Mmapper.load_file("file1.txt")
+m2 = Mmapper.load_file("file2.txt")
+
+puts m1.find_matching_line("something")
+puts m2.find_matching_line("another")
+```
+
+## Running Tests
+There's a small test script:
+
+```sh
+ruby test_mmapper.rb
+```
+
+## License
+This project is licensed under the MIT License.
+

data/ext/extconf.rb

@@ -0,0 +1,29 @@
+require 'mkmf'
+require 'fileutils'
+
+LIB_DIR = File.expand_path(File.dirname(__FILE__))
+
+# Detect OS and set correct library name
+LIB_NAME =
+  case RUBY_PLATFORM
+  when /darwin/ then "libmmapper.dylib"
+  when /mingw|mswin/ then "mmapper.dll"
+  else "libmmapper.so"
+  end
+
+LIB_PATH = File.join(LIB_DIR, LIB_NAME)
+
+# Ensure Go module is initialized
+Dir.chdir(LIB_DIR) do
+  unless File.exist?("go.mod")
+    puts "Initializing Go module..."
+    system("go mod init mmapper") || raise("Failed to initialize Go module")
+  end
+
+  puts "Building Go shared library for #{RUBY_PLATFORM}..."
+  unless system("go build -o #{LIB_NAME} -buildmode=c-shared .")
+    raise "Go build failed!"
+  end
+end
+
+create_makefile('mmapper')

data/ext/mmap.go

@@ -0,0 +1,127 @@
+package main
+
+/*
+#include <stdlib.h>
+*/
+import "C"
+
+import (
+	"bytes"
+	"os"
+	"sync"
+	"syscall"
+)
+
+// Mmapper holds the mmap data for a file
+type Mmapper struct {
+	mmapData []byte
+	fileSize int
+}
+
+// Store instances
+var (
+	mu        sync.Mutex
+	mmappers  = make(map[int]*Mmapper)
+	nextID    = 1
+)
+
+// mmapFile maps a file into memory.
+func mmapFile(filename string) (*Mmapper, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	size := fi.Size()
+	if size == 0 {
+		return nil, os.ErrInvalid
+	}
+
+	data, err := syscall.Mmap(int(file.Fd()), 0, int(size), syscall.PROT_READ, syscall.MAP_SHARED)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Mmapper{mmapData: data, fileSize: int(size)}, nil
+}
+
+// findLineStart moves backward to find the start of a line.
+func findLineStart(data []byte, pos int) int {
+	for pos > 0 && data[pos-1] != '\n' {
+		pos--
+	}
+	return pos
+}
+
+// readLine reads a full line starting from a given position.
+func readLine(data []byte, start int, fileSize int) string {
+	end := start
+	for end < fileSize && data[end] != '\n' {
+		end++
+	}
+	return string(data[start:end])
+}
+
+// binarySearchPrefix performs a binary search for a prefix.
+func binarySearchPrefix(m *Mmapper, prefix string) string {
+	low, high := 0, m.fileSize-1
+	var match string
+
+	for low <= high {
+		mid := (low + high) / 2
+		mid = findLineStart(m.mmapData, mid)
+
+		line := readLine(m.mmapData, mid, m.fileSize)
+		if bytes.HasPrefix([]byte(line), []byte(prefix)) {
+			match = line
+			high = mid - 1
+		} else if line < prefix {
+			low = mid + len(line) + 1
+		} else {
+			high = mid - 1
+		}
+	}
+	return match
+}
+
+//export CreateMmapper
+func CreateMmapper(filename *C.char) C.int {
+	m, err := mmapFile(C.GoString(filename))
+	if err != nil {
+		return -1 // Error case
+	}
+
+	mu.Lock()
+	id := nextID
+	nextID++
+	mmappers[id] = m
+	mu.Unlock()
+
+	return C.int(id)
+}
+
+//export FindMatchingLine
+func FindMatchingLine(mmapperID C.int, prefix *C.char) *C.char {
+	mu.Lock()
+	m, exists := mmappers[int(mmapperID)]
+	mu.Unlock()
+
+	if !exists {
+		return nil
+	}
+
+	match := binarySearchPrefix(m, C.GoString(prefix))
+	if match == "" {
+		return nil
+	}
+
+	return C.CString(match)
+}
+
+func main() {}

data/lib/mmapper.rb

@@ -0,0 +1,45 @@
+require 'ffi'
+
+module Mmapper
+  extend FFI::Library
+
+  LIB_DIR = File.expand_path("../../ext", __FILE__)
+
+  LIB_NAME =
+    case RUBY_PLATFORM
+    when /darwin/ then "libmmapper.dylib"
+    when /mingw|mswin/ then "mmapper.dll"
+    else "libmmapper.so"
+    end
+
+  LIB_PATH = File.join(LIB_DIR, LIB_NAME)
+
+  unless File.exist?(LIB_PATH)
+    Dir.chdir(LIB_DIR) do
+      puts "Compiling Go shared library for #{RUBY_PLATFORM}..."
+      system("go build -o #{LIB_NAME} -buildmode=c-shared .") || raise("Go build failed")
+    end
+  end
+
+  ffi_lib LIB_PATH
+
+  attach_function :create_mmapper, :CreateMmapper, [:string], :int
+  attach_function :find_matching_line, :FindMatchingLine, [:int, :string], :string
+
+  class Instance
+    def initialize(filename)
+      @mmapper_id = Mmapper.create_mmapper(filename)
+      raise "Failed to load file: #{filename}" if @mmapper_id < 0
+    end
+
+    def find_matching_line(prefix)
+      result = Mmapper.find_matching_line(@mmapper_id, prefix)
+      return nil if result.nil? || result.empty?
+      result
+    end
+  end
+
+  def self.load_file(filename)
+    Instance.new(filename)
+  end
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: mmapper
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Carl Dawson
+bindir: bin
+cert_chain: []
+date: 2025-01-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+description: Wraps a Go extension for mmap-ing files.
+email:
+- email@carldaws.com
+executables: []
+extensions:
+- ext/extconf.rb
+extra_rdoc_files: []
+files:
+- README.md
+- ext/extconf.rb
+- ext/mmap.go
+- lib/mmapper.rb
+homepage: https://github.com/carldaws/mmapper
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.2
+specification_version: 4
+summary: Mmap-ed files using Go and FFI.
+test_files: []