funnel_http 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cc6d1f5ffef3ad3e41de07eb3b94342bd082724aa3d1daad338603fbedbabcdc
+  data.tar.gz: f3208ba0c9154bd99fd2d373e91d3679690f0f711c74a3f11038c500c08fbd8a
+SHA512:
+  metadata.gz: 87f9ad18e8c0005f2a3520d06381f3e308bbf6372827b631d646546c48f929be5879da1296331d9bab4ec7667786ce353473930b6795b84d0a46c11f9e68ec12
+  data.tar.gz: c92e03744f09aa670972e188bb0c94db2b2920f028994db5bb0f26d3804d470345b43087d57195666d1236bfc2adcc9c8ee8ccde4c0ca50ae94b54887a0c1dd6

data/.golangci.yml

@@ -0,0 +1,21 @@
+linters-settings:
+  gofmt:
+  revive:
+    rules:
+      - name: exported
+        arguments:
+          - disableStutteringCheck
+  testifylint:
+  wrapcheck:
+
+linters:
+  enable:
+    - gofmt
+    - revive
+    - testifylint
+    - wrapcheck
+
+issues:
+  include:
+    - EXC0012 # EXC0012 revive: Annoying issue about not having a comment. The rare codebase has such comments
+    - EXC0014 # EXC0014 revive: Annoying issue about not having a comment. The rare codebase has such comments

data/.rspec

@@ -0,0 +1,3 @@
+--format progress
+--color
+--require spec_helper

data/.yardopts

@@ -0,0 +1,6 @@
+--markup markdown
+--no-private
+--hide-void-return
+-
+CHANGELOG.md
+LICENSE.txt

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+## [Unreleased]
+[full changelog](http://github.com/sue445/funnel_http/compare/v0.1.0...main)
+
+## [0.1.0](https://github.com/sue445/funnel_http/releases/tag/v0.1.0) - 2024-12-18
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 sue445
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,87 @@
+# FunnelHttp
+Perform HTTP requests in parallel
+
+[![build](https://github.com/sue445/funnel_http/actions/workflows/build.yml/badge.svg)](https://github.com/sue445/funnel_http/actions/workflows/build.yml)
+
+## Requirements
+* Ruby
+* Go
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+```bash
+bundle add funnel_http
+```
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+```bash
+gem install funnel_http
+```
+
+## Usage
+Use [`FunnelHttp::Client#perform`](https://sue445.github.io/funnel_http/FunnelHttp/Client.html#perform-instance_method)
+
+```ruby
+require "funnel_http"
+
+client = FunnelHttp::Client.new
+
+requests = [
+  {
+    method: :get,
+    uri: "https://example.com/api/user/1",
+  },
+
+  # with request header
+  { 
+    method: :get, 
+    uri: "https://example.com/api/user/2", 
+    header: {
+      "Authorization" => "Bearer xxxxxxxx",
+      "X-Multiple-Values" => ["1st value", "2nd value"],
+    },
+  },
+]
+
+responses = client.perform(requests)
+# => [
+#   { status_code: 200, body: "Response of /api/user/1", header: { "Content-Type" => ["text/plain;charset=utf-8"]} }
+#   { status_code: 200, body: "Response of /api/user/2", header: { "Content-Type" => ["text/plain;charset=utf-8"]} }
+# ]
+```
+
+## Customize
+### Add default header
+Example1. Pass default header to [`FunnelHttp::Client#initialize`](https://sue445.github.io/funnel_http/FunnelHttp/Client.html#normalize_requests-instance_method)
+
+```ruby
+default_header = { "Authorization" => "Bearer xxxxxx" }
+
+client = FunnelHttp::Client.new(default_header)
+```
+
+Example 2. Use [`FunnelHttp::Client#add_default_request_header`](https://sue445.github.io/funnel_http/FunnelHttp/Client.html#add_default_request_header-instance_method)
+
+```ruby
+client.add_default_request_header("Authorization", "Bearer xxxxxx")
+```
+
+## API Reference
+https://sue445.github.io/funnel_http/
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/sue445/funnel_http.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rake/extensiontask"
+
+task build: :compile
+
+GEMSPEC = Gem::Specification.load("funnel_http.gemspec")
+
+Rake::ExtensionTask.new("funnel_http", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/funnel_http"
+end
+
+require "go_gem/rake_task"
+
+go_task = GoGem::RakeTask.new("funnel_http")
+
+namespace :go do
+  desc "Run golangci-lint"
+  task :lint do
+    go_task.within_target_dir do
+      sh "which golangci-lint" do |ok, _|
+        raise "golangci-lint isn't installed. See. https://golangci-lint.run/welcome/install/" unless ok
+      end
+      sh GoGem::RakeTask.build_env_vars, "golangci-lint run"
+    end
+  end
+
+  desc "Run go mod tidy"
+  task :mod_tidy do
+    go_task.within_target_dir do
+      sh "go mod tidy"
+    end
+  end
+end
+
+namespace :rbs do
+  desc "`rbs collection install` and `git commit`"
+  task :install do
+    sh "rbs collection install"
+    sh "git add rbs_collection.lock.yaml"
+    sh "git commit -m 'rbs collection install' || true"
+  end
+end
+
+desc "Check rbs"
+task :rbs do
+  sh "rbs validate"
+  sh "steep check"
+end
+
+task default: %i[clobber compile go:test spec]

data/Steepfile

@@ -0,0 +1,31 @@
+# D = Steep::Diagnostic
+
+target :lib do
+  signature "sig"
+
+  check "lib"                       # Directory name
+  # check "Gemfile"                   # File name
+  # check "app/models/**/*.rb"        # Glob
+  # ignore "lib/templates/*.rb"
+
+  # library "pathname"              # Standard libraries
+  # library "strong_json"           # Gems
+
+  collection_config "rbs_collection.yaml"
+
+  # configure_code_diagnostics(D::Ruby.default)      # `default` diagnostics setting (applies by default)
+  # configure_code_diagnostics(D::Ruby.strict)       # `strict` diagnostics setting
+  # configure_code_diagnostics(D::Ruby.lenient)      # `lenient` diagnostics setting
+  # configure_code_diagnostics(D::Ruby.silent)       # `silent` diagnostics setting
+  # configure_code_diagnostics do |hash|             # You can setup everything yourself
+  #   hash[D::Ruby::NoMethod] = :information
+  # end
+end
+
+# target :test do
+#   signature "sig", "sig-private"
+#
+#   check "test"
+#
+#   # library "pathname"              # Standard libraries
+# end

data/benchmark/README.md

@@ -0,0 +1,32 @@
+# benchmark for funnel_http
+## Usage
+```bash
+docker compose up --build
+```
+
+```bash
+bundle exec ruby benchmark.rb
+```
+
+## Report
+```
+Warming up --------------------------------------
+FunnelHttp::Client#perform
+                         2.000 i/100ms
+Parallel with 4 processes
+                         1.000 i/100ms
+Parallel with 4 threads
+                         1.000 i/100ms
+Calculating -------------------------------------
+FunnelHttp::Client#perform
+                         21.816 (± 4.6%) i/s   (45.84 ms/i) -     44.000 in   2.026960s
+Parallel with 4 processes
+                         15.785 (± 6.3%) i/s   (63.35 ms/i) -     32.000 in   2.035628s
+Parallel with 4 threads
+                         18.570 (±10.8%) i/s   (53.85 ms/i) -     37.000 in   2.008485s
+
+Comparison:
+FunnelHttp::Client#perform:       21.8 i/s
+Parallel with 4 threads:       18.6 i/s - 1.17x  slower
+Parallel with 4 processes:       15.8 i/s - 1.38x  slower
+```

data/benchmark/benchmark.rb

@@ -0,0 +1,61 @@
+require "benchmark/ips"
+require "open-uri"
+require "parallel"
+require "etc"
+
+ROOT_DIR = File.expand_path("..", __dir__)
+
+TEST_SERVER_URL = "http://localhost:8080/"
+
+REQUEST_COUNT = 100
+
+BENCHMARK_CONCURRENCY = ENV.fetch("BENCHMARK_CONCURRENCY") { 4 }
+
+# Build native extension before running benchmark
+Dir.chdir(ROOT_DIR) do
+  system("bundle config set --local path 'vendor/bundle'", exception: true)
+  system("bundle install", exception: true)
+  system("bundle exec rake clobber compile", exception: true)
+end
+
+require_relative "../lib/funnel_http"
+
+# Suppress Ractor warning
+$VERBOSE = nil
+
+system("go version", exception: true)
+
+requests = Array.new(REQUEST_COUNT, { method: :get, url: TEST_SERVER_URL })
+
+def fetch_server
+  URI.parse(TEST_SERVER_URL).open(open_timeout: 90, read_timeout: 90).read
+end
+
+Benchmark.ips do |x|
+  x.config(warmup: 1, time: 2)
+
+  x.report("FunnelHttp::Client#perform") do
+    FunnelHttp::Client.new.perform(requests)
+  end
+
+  x.report("Parallel with #{BENCHMARK_CONCURRENCY} processes") do
+    Parallel.each(requests, in_processes: BENCHMARK_CONCURRENCY) do
+      fetch_server
+    end
+  end
+
+  x.report("Parallel with #{BENCHMARK_CONCURRENCY} threads") do
+    Parallel.each(requests, in_threads: BENCHMARK_CONCURRENCY) do
+      fetch_server
+    end
+  end
+
+  # FIXME: open-uri doesn't work in Ractor
+  # x.report("Parallel with Ractor") do
+  #   REQUEST_COUNT.times.map do
+  #     Ractor.new { URI.parse("http://localhost:8080/").read }
+  #   end.each(&:take)
+  # end
+
+  x.compare!
+end

data/benchmark/compose.yml

@@ -0,0 +1,8 @@
+services:
+  nginx:
+    image: nginx:latest
+    ports:
+      - "8080:80"
+    volumes:
+      - ./html:/usr/share/nginx/html
+      - ./nginx.conf:/etc/nginx/nginx.conf

data/benchmark/html/index.html

@@ -0,0 +1,4 @@
+<html>
+<head></head>
+<body>it works</body>
+</html>

data/benchmark/nginx.conf

@@ -0,0 +1,25 @@
+worker_processes auto;
+worker_rlimit_nofile 12288;
+
+events {
+  worker_connections 4096;
+}
+
+http {
+  include       mime.types;
+  default_type  application/octet-stream;
+
+  sendfile        on;
+  keepalive_timeout  65;
+  access_log  off;
+
+  server {
+    listen 80;
+    server_name localhost;
+
+    location / {
+      root /usr/share/nginx/html;
+      index index.html;
+    }
+  }
+}

data/ext/funnel_http/extconf.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "go_gem/mkmf"
+
+# Makes all symbols private by default to avoid unintended conflict
+# with other gems. To explicitly export symbols you can use RUBY_FUNC_EXPORTED
+# selectively, or entirely remove this flag.
+append_cflags("-fvisibility=hidden")
+
+create_go_makefile("funnel_http/funnel_http")

data/ext/funnel_http/funnel_http.c

@@ -0,0 +1,2 @@
+#include "funnel_http.h"
+#include "_cgo_export.h"

data/ext/funnel_http/funnel_http.go

@@ -0,0 +1,164 @@
+package main
+
+/*
+#include "funnel_http.h"
+
+VALUE rb_go_data_alloc(VALUE klass);
+VALUE rb_funnel_http_run_requests(VALUE self, VALUE rbAry);
+*/
+import "C"
+
+import (
+	"github.com/ruby-go-gem/go-gem-wrapper/ruby"
+	"net/http"
+	"unsafe"
+)
+
+//export rb_funnel_http_run_requests
+func rb_funnel_http_run_requests(self C.VALUE, rbAry C.VALUE) C.VALUE {
+	rbAryLength := int(ruby.RARRAY_LEN(ruby.VALUE(rbAry)))
+	requests := make([]Request, 0, rbAryLength)
+
+	for i := 0; i < rbAryLength; i++ {
+		rbHash := ruby.RbAryEntry(ruby.VALUE(rbAry), ruby.Long(i))
+
+		req := Request{
+			Method: getRbHashValueAsString(rbHash, "method"),
+			URL:    getRbHashValueAsString(rbHash, "url"),
+			Header: getRbHashValueAsMap(rbHash, "header"),
+		}
+		requests = append(requests, req)
+	}
+
+	httpClient := getHttpClientFromInstanceVariable(ruby.VALUE(self))
+
+	responses, err := RunRequests(&httpClient, requests)
+	if err != nil {
+		ruby.RbRaise(rb_cFunnelHttpError, "%s", err.Error())
+	}
+
+	var rbHashSlice []ruby.VALUE
+	for _, response := range responses {
+		rbHash := ruby.RbHashNew()
+
+		ruby.RbHashAset(rbHash, ruby.RbId2Sym(ruby.RbIntern("status_code")), ruby.INT2NUM(response.StatusCode))
+		ruby.RbHashAset(rbHash, ruby.RbId2Sym(ruby.RbIntern("body")), ruby.String2Value(string(response.Body)))
+
+		rbHashHeader := ruby.RbHashNew()
+		ruby.RbGcRegisterAddress(&rbHashHeader)
+		defer ruby.RbGcUnregisterAddress(&rbHashHeader)
+
+		for key, values := range response.Header {
+			var headerValues []ruby.VALUE
+			for _, value := range values {
+				v := ruby.String2Value(value)
+				ruby.RbGcRegisterAddress(&v)
+				defer ruby.RbGcUnregisterAddress(&v)
+
+				headerValues = append(headerValues, v)
+			}
+			k := ruby.String2Value(key)
+			ruby.RbGcRegisterAddress(&k)
+			defer ruby.RbGcUnregisterAddress(&k)
+
+			v := ruby.Slice2rbAry(headerValues)
+			ruby.RbGcRegisterAddress(&v)
+			defer ruby.RbGcUnregisterAddress(&v)
+
+			ruby.RbHashAset(rbHashHeader, k, v)
+		}
+		ruby.RbHashAset(rbHash, ruby.RbId2Sym(ruby.RbIntern("header")), rbHashHeader)
+
+		rbHashSlice = append(rbHashSlice, rbHash)
+	}
+
+	return C.VALUE(ruby.Slice2rbAry(rbHashSlice))
+}
+
+func getHttpClientFromInstanceVariable(self ruby.VALUE) http.Client {
+	id := ruby.RbIntern("@__go_data")
+	value := ruby.RbIvarGet(self, id)
+
+	if !ruby.RB_NIL_P(value) {
+		// return http.Client in instance variable
+		data := (*goData)(ruby.GetGoStruct(value))
+		return data.httpClient
+	}
+
+	// Create instance of FunnelHttp::Ext::GoData
+	obj := ruby.RbObjAlloc(rb_cFunnelHttpExtGoData)
+	data := ruby.GetGoStruct(obj)
+
+	// Save FunnelHttp::Ext::GoData to instance variable of FunnelHttp::Ext::Client
+	dataValue := (*ruby.VALUE)(data)
+	ruby.RbIvarSet(self, id, *dataValue)
+
+	return ((*goData)(data)).httpClient
+}
+
+type goData struct {
+	httpClient http.Client
+}
+
+//export rb_go_data_alloc
+func rb_go_data_alloc(klass C.VALUE) C.VALUE {
+	data := goData{}
+	return C.VALUE(ruby.NewGoStruct(ruby.VALUE(klass), unsafe.Pointer(&data)))
+}
+
+func getRbHashValueAsString(rbHash ruby.VALUE, key string) string {
+	value := ruby.RbHashAref(rbHash, ruby.RbToSymbol(ruby.String2Value(key)))
+	return ruby.Value2String(value)
+}
+
+func getRbHashValueAsMap(rbHash ruby.VALUE, key string) map[string][]string {
+	rbHashValue := ruby.RbHashAref(rbHash, ruby.RbToSymbol(ruby.String2Value(key)))
+	rbKeys := ruby.CallFunction(rbHashValue, "keys")
+
+	var ret = map[string][]string{}
+
+	for i := 0; i < int(ruby.RARRAY_LEN(rbKeys)); i++ {
+		rbKey := ruby.RbAryEntry(rbKeys, ruby.Long(i))
+		rbAryValue := ruby.RbHashAref(rbHashValue, rbKey)
+
+		var values []string
+		for j := 0; j < int(ruby.RARRAY_LEN(rbAryValue)); j++ {
+			str := ruby.Value2String(ruby.RbAryEntry(rbAryValue, ruby.Long(j)))
+			values = append(values, str)
+		}
+
+		keyName := ruby.Value2String(rbKey)
+		ret[keyName] = values
+	}
+
+	return ret
+}
+
+// revive:disable:exported
+
+var rb_cFunnelHttpError ruby.VALUE
+var rb_cFunnelHttpExtGoData ruby.VALUE
+
+//export Init_funnel_http
+func Init_funnel_http() {
+	rb_mFunnelHttp := ruby.RbDefineModule("FunnelHttp")
+
+	// FunnelHttp::Ext
+	rb_mFunnelHttpExt := ruby.RbDefineModuleUnder(rb_mFunnelHttp, "Ext")
+
+	// FunnelHttp::Ext::Client
+	rb_cFunnelHttpExtClient := ruby.RbDefineClassUnder(rb_mFunnelHttpExt, "Client", ruby.VALUE(C.rb_cObject))
+	ruby.RbDefineMethod(rb_cFunnelHttpExtClient, "run_requests", C.rb_funnel_http_run_requests, 1)
+
+	// FunnelHttp::Ext::GoData
+	rb_cFunnelHttpExtGoData = ruby.RbDefineClassUnder(rb_mFunnelHttpExt, "GoData", ruby.VALUE(C.rb_cObject))
+	ruby.RbDefineAllocFunc(rb_cFunnelHttpExtGoData, C.rb_go_data_alloc)
+
+	// FunnelHttp::Error
+	rb_cFunnelHttpError = ruby.RbDefineClassUnder(rb_mFunnelHttp, "Error", ruby.VALUE(C.rb_eStandardError))
+}
+
+// revive:enable:exported
+
+func main() {
+}

data/ext/funnel_http/funnel_http.h

@@ -0,0 +1,6 @@
+#ifndef FUNNEL_HTTP_H
+#define FUNNEL_HTTP_H 1
+
+#include "ruby.h"
+
+#endif /* FUNNEL_HTTP_H */

data/ext/funnel_http/go.mod

@@ -0,0 +1,27 @@
+module github.com/sue445/funnel_http
+
+go 1.23
+
+require (
+	github.com/cockroachdb/errors v1.11.3
+	github.com/jarcoal/httpmock v1.3.1
+	github.com/ruby-go-gem/go-gem-wrapper v0.5.1
+	github.com/stretchr/testify v1.10.0
+	golang.org/x/sync v0.10.0
+)
+
+require (
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/getsentry/sentry-go v0.27.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)