encoded_id 1.0.0.rc4 → 1.0.0.rc5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0844f07baeecd48a70bf0b2564dfb9f02998b002279b44bf75bd0172ccc265be'
-  data.tar.gz: 0bcd81de8a4ea7e034097a92dff9ba82478d92f87f7820dda872a9f35b5294d3
+  metadata.gz: b93ebfafa557cd45d9158fa850a474cb3f56de3255b650aeed11bbc131b67ed2
+  data.tar.gz: a3a9feb5004a637c6190ea3d969e03de6066d3b94e7b9b54d8be1c639b8b17af
 SHA512:
-  metadata.gz: 4dcf3c99891f8a3a195d188656d1c45c9471760dfbc715fe994fac4c4145a086d9573b663c5a4a79894c71c4e3a15bf68ff0d30b20e44b8fe661c68f1707c065
-  data.tar.gz: 8edb3675d565ff7be88441fcdb2a55783415f170fcd8d9102c6bb0fff8cc6fa0d22f1af19e72dd5558dedb37ba2af23c29fd6171d12da2032a0e2ce187bedeca
+  metadata.gz: 3f06797187a84df0d6436868d82d9d096220e88e9087bad13a78cc766e09681ffdd1db73daa7f9af90288e988b0e4034881b1fc0f7c9129bc6b64a9b8e332589
+  data.tar.gz: 764f505dfb1209bb358784906cfc648bdfee6cf58d79fb5872105d1234ff2839d072b3e899176f4e765f69caccabc00d9d105c9bf484ce2ccf170fdc594c7f44

data/.devcontainer/Dockerfile

@@ -0,0 +1,9 @@
+# Make sure RUBY_VERSION matches the Ruby version in .ruby-version or gemspec
+ARG RUBY_VERSION=3.4.2
+FROM ghcr.io/rails/devcontainer/images/ruby:$RUBY_VERSION
+
+USER vscode
+
+# Ensure binding is always 0.0.0.0
+# Binds the server to all IP addresses of the container, so it can be accessed from outside the container.
+ENV BINDING="0.0.0.0"

data/.devcontainer/compose.yml

@@ -0,0 +1,8 @@
+name: "encoded_id"
+
+services:
+  encoded-id-dev-env:
+    container_name: encoded-id-dev-env
+    build:
+      context: ..
+      dockerfile: .devcontainer/Dockerfile

data/.devcontainer/devcontainer.json

@@ -0,0 +1,8 @@
+{
+  "name": "Encoded ID Gem Development",
+  "dockerComposeFile": "compose.yml",
+  "service": "encoded-id-dev-env",
+  "postCreateCommand": "bundle install",
+  "postStartCommand": "bundle exec rake test",
+  "remoteUser": "vscode"
+}

data/.standard.yml

@@ -0,0 +1,2 @@
+parallel: true          # default: false
+ruby_version: 3.3

data/CHANGELOG.md

@@ -4,11 +4,18 @@
 
 ### Breaking changes
 
-- `ReversibleId` now no longer downcases the encodedid input string by default on decode, ie the `decode` option `downcase` is now `false`. In a future release the `downcase` option will be removed.
+- `ReversibleId` now no longer downcases the encodedid input string by default on decode, ie the `decode` option `downcase` is now `false`. In a future release the `downcase` option will be removed. Generation of the encoded ID is 1.5 times faster and uses less memory.
 
-## [1.0.0.rc4] - unreleased
+## [1.0.0] - unreleased
+
+## [1.0.0.rc5] - 2025-04-09
+
+- `encoded_id` now uses its own implementation of `hashids` which is more efficient and has a smaller memory footprint. This massively reduces the GC churn in high-throughput applications. This is an implementation based on the original `hashids` gem but with many optimisations and improvements. Functionally it is identical to the original `hashids` gem.
+
+## [1.0.0.rc4] - 2024-04-29
 
 - Add an optional `max_inputs_per_id` argument to `ReversibleId`, thanks to [@avcwisesa](https://github.com/avcwisesa)
+- The option `split_with:` can also now be set to nil to disable splitting of the encoded ID string
 
 ## [1.0.0.rc3] - 2023-10-23
 

data/Gemfile

@@ -5,15 +5,15 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in encoded_id.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
+gem "rake"
 
 gem "minitest"
 
-gem "standard", "~> 1.30"
+gem "standard"
 
-gem "rbs"
-
-gem "steep"
+# gem "rbs"
+#
+# gem "steep"
 
 gem "simplecov"
 
@@ -22,3 +22,15 @@ gem "benchmark-ips"
 gem "benchmark-memory"
 
 gem "fuzzbert"
+
+gem "singed"
+
+gem "memory_profiler"
+
+gem "hashids" # For benchmarking against
+
+gem "base64"
+
+# gem "pf2", require: false, github: "osyoyu/pf2", branch: "main"
+
+# gem "vernier", require: false

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2022 Stephen Ierodiaconou
+Copyright (c) 2022-2024 Stephen Ierodiaconou
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -33,7 +33,7 @@ coder.decode("z2j7-Odmw") # (note the capital 'o' instead of zero)
 
 ## Features
 
-* 🔄 encoded IDs are reversible (uses with https://hashids.org))
+* 🔄 encoded IDs are reversible (uses Hashids, the old site is here https://github.com/hashids/hashids.github.io))
 * 👥 supports multiple IDs encoded in one encoded string (eg `7aq6-0zqw` decodes to `[78, 45]`)
 * 🔡 supports custom alphabets for the encoded string (at least 16 characters needed)
   - by default uses a variation of the Crockford reduced character set (https://www.crockford.com/base32.html)
@@ -51,9 +51,12 @@ I aim for 100% test coverage and have fuzz tested quite extensively. But please
 
 * support for encoding of hex strings (eg UUIDs), including multiple IDs encoded in one string
 
-### Coming soon
+### Performance and benchmarking
+
+This gem uses a custom HashId implementation that is significantly faster and more memory-efficient than the original `hashids` gem. 
+
+For detailed benchmarks and performance metrics, see the [Custom HashId Implementation](#custom-hashid-implementation) section at the end of this README.
 
-Performance improvements and benchmarking!
 
 ### Rails support `encoded_id-rails`
 
@@ -208,6 +211,8 @@ For readability, the encoded string can be split into groups of characters.
 
 `split_with`: specifies the separator to use between the groups. Default is `-`.
 
+Set either to `nil` to disable splitting.
+
 ### `hex_digit_encoding_group_size`
 
 **Experimental**
@@ -333,5 +338,47 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/steveg
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
 
+## Custom HashId Implementation
+
+Internally, `encoded_id` uses its own HashId implementation (`EncodedId::HashId`) instead of the original `hashids` gem. This custom implementation was created to improve both performance and memory usage. 
+
+Recent benchmarks show significant improvements:
+
+### Performance Comparison
+
+```
+| Test                      | Hashids (i/s) | EncodedId::HashId (i/s) | Speedup |
+| ------------------------- | ------------ | --------------------- | ------- |
+| #encode - 1 ID            |  131,000.979 |           197,586.231 |   1.51x |
+| #decode - 1 ID            |   65,791.334 |            92,425.571 |   1.40x |
+| #encode - 10 IDs          |   13,773.355 |            20,669.715 |   1.50x |
+| #decode - 10 IDs          |    6,911.872 |             9,990.078 |   1.45x |
+| #encode w YJIT - 1 ID     |  265,764.969 |           877,551.362 |   3.30x |
+| #decode w YJIT - 1 ID     |  130,154.837 |           348,000.817 |   2.67x |
+| #encode w YJIT - 10 IDs   |   27,966.457 |           100,461.237 |   3.59x |
+| #decode w YJIT - 10 IDs   |   14,187.346 |            43,974.011 |   3.10x |
+| #encode w YJIT - 1000 IDs |      268.140 |             1,077.855 |   4.02x |
+| #decode w YJIT - 1000 IDs |      136.217 |               464.579 |   3.41x |
+```
+
+With YJIT enabled, the performance improvements are even more significant, with up to 4x faster operation for large inputs.
+
+### Memory Usage Comparison
+
+```
+| Test                | Implementation   | Allocated Memory | Allocated Objects | Memory Reduction |
+| ------------------- | ---------------- | ---------------- | ----------------- | ---------------- |
+| encode small input  | Hashids          |          7.28 KB |               120 |                - |
+|                     | EncodedId::HashId |            920 B |                 6 |           87.66% |
+| encode large input  | Hashids          |        403.36 KB |              5998 |                - |
+|                     | EncodedId::HashId |          8.36 KB |               104 |           97.93% |
+| decode large input  | Hashids          |        366.88 KB |              5761 |                - |
+|                     | EncodedId::HashId |         14.63 KB |               264 |           96.01% |
+```
+
+The memory usage improvements are dramatic, with up to 98% reduction in memory allocation for large inputs.
+
+Run `bin/are_we_fast_yet` and `bin/memory_profile` in your environment to see the current performance difference.
+
 ## keywords
 hash ID, friendly ID, obfuscate ID, rails, ActiveRecord, model, slug, vanity URL, friendly URL

data/Rakefile

@@ -9,6 +9,12 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/test_*.rb"]
 end
 
-require "standard/rake"
-
 task default: %i[test standard]
+
+task :compile_ext do
+  puts "Compiling extension"
+  `cd ext/encoded_id && make clean`
+  `cd ext/encoded_id && ruby extconf.rb`
+  `cd ext/encoded_id && make`
+  puts "Done"
+end

data/ext/encoded_id/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile "encoded_id/extension"

data/ext/encoded_id/extension.c

@@ -0,0 +1,123 @@
+#include "ruby/ruby.h"
+#include "hashids.h"
+
+void wrapped_hashids_free(void* data)
+{
+    hashids_free(data);
+}
+
+size_t wrapped_hashids_size(const void* data)
+{
+    return sizeof(hashids_t);
+}
+
+static const rb_data_type_t wrapped_hashids_type = {
+    .wrap_struct_name = "hashids_t",
+    .function = {
+        .dmark = NULL,
+        .dfree = wrapped_hashids_free,
+        .dsize = wrapped_hashids_size,
+    },
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+};
+
+VALUE hashids_alloc(VALUE self)
+{
+    hashids_t *data = hashids_init("salt!");
+    return TypedData_Wrap_Struct(self, &wrapped_hashids_type, data);
+}
+
+//VALUE rb_hashids_m_initialize(VALUE self, VALUE val)
+//{
+//    return self;
+//}
+
+
+static VALUE rb_hash_id_c_encode(VALUE self, VALUE ids) {
+    Check_Type(ids, T_ARRAY);
+
+    long length = RARRAY_LEN(ids);
+
+    unsigned long long* inputs = ALLOC_N(unsigned long long, length);
+
+    for (long i = 0; i < length; i++) {
+        VALUE rb_element = rb_ary_entry(ids, i);
+        Check_Type(rb_element, T_FIXNUM);
+        inputs[i] = NUM2ULL(rb_element);
+    }
+
+    hashids_t* hashids;
+
+    TypedData_Get_Struct(self, hashids_t, &wrapped_hashids_type, hashids);
+
+    size_t bytes_encoded;
+
+    size_t bytes_needed;
+    bytes_needed = hashids_estimate_encoded_size(hashids, sizeof(&inputs) / sizeof(unsigned long long), &inputs);
+    char *hash = ALLOC_N(char, bytes_needed);
+
+    // unsigned long long numbers[] = {1ull, 2ull, 3ull, 4ull, 5ull};
+
+//    printf("length: %ld\n", length);
+//    printf("inputs[0]: %llu\n", inputs[0]);
+//    printf("inputs[1]: %llu\n", inputs[1]);
+//    printf("inputs[2]: %llu\n", inputs[2]);
+//    printf("inputs[3]: %llu\n", inputs[3]);
+//    printf("inputs[4]: %llu\n", inputs[4]);
+//
+//    printf("hashids: %p\n", hashids);
+//    printf("hashids->alphabet: %s\n", hashids->alphabet);
+//    printf("hashids->salt: %s\n", hashids->salt);
+//    printf("hashids->min_hash_length: %lu\n", hashids->min_hash_length);
+//    printf("numbers: %p\n", numbers);
+//    printf("numbers[0]: %llu\n", numbers[0]);
+//    printf("numbers[1]: %llu\n", numbers[1]);
+//    printf("numbers[2]: %llu\n", numbers[2]);
+//    printf("numbers[3]: %llu\n", numbers[3]);
+//    printf("numbers[4]: %llu\n", numbers[4]);
+//
+//    printf("sizeof(*inputs) / sizeof(unsigned long long): %lu\n", sizeof(*inputs) / sizeof(unsigned long long));
+//    printf("sizeof(numbers) / sizeof(unsigned long long): %lu\n", sizeof(numbers) / sizeof(unsigned long long));
+    // bytes_encoded = hashids_encode(hashids, hash, sizeof(numbers) / sizeof(unsigned long long), numbers);
+    bytes_encoded = hashids_encode(hashids, hash, length, inputs);
+
+    ruby_xfree(inputs);
+    VALUE return_value = rb_str_new2(hash);
+    ruby_xfree(hash);
+    return return_value;
+}
+
+static VALUE rb_hash_id_c_decode(VALUE self, VALUE str) {
+    Check_Type(str, T_STRING);
+
+    hashids_t* hashids;
+
+    TypedData_Get_Struct(self, hashids_t, &wrapped_hashids_type, hashids);
+
+    size_t numbers_count = hashids_numbers_count(hashids, RSTRING_PTR(str));
+
+    unsigned long long* numbers = ALLOC_N(unsigned long long, numbers_count);
+
+    hashids_decode_safe(hashids, RSTRING_PTR(str), numbers, numbers_count);
+
+    VALUE rb_numbers = rb_ary_new_capa(numbers_count);
+
+    for (size_t i = 0; i < numbers_count; i++) {
+        rb_ary_push(rb_numbers, ULL2NUM(numbers[i]));
+    }
+
+    ruby_xfree(numbers);
+    return rb_numbers;
+}
+
+void Init_extension(void) {
+    VALUE EncodedId = rb_define_module("EncodedId");
+    VALUE HashIdC = rb_define_class_under(EncodedId, "HashIdC", rb_cObject);
+
+    rb_define_alloc_func(HashIdC, hashids_alloc);
+//    rb_define_method(HashIdC, "initialize", rb_hashids_m_initialize, 1);
+
+    rb_define_method(HashIdC, "encode", rb_hash_id_c_encode, 1);
+    rb_define_method(HashIdC, "decode", rb_hash_id_c_decode, 1);
+}