wayback_machine_downloader 0.1.11 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f92d7e391faeb42397b9977342485cb66f63fda
-  data.tar.gz: a9c72cc1304cd28226f6fcd111cba6398ab3f5b0
+  metadata.gz: d6580398142a2bde95000f07ef0063927b969676
+  data.tar.gz: 029c7e07d34b4c2b3b7d0c85c0567d495c6bd4ed
 SHA512:
-  metadata.gz: 700e7774978a1b2f02eac2485126fe9e8f900a741a45c1d30ab6a89c890029c7d04ec59938147181392a24b663c43cbdd878865a553dfbd5a430abaaa2d8899a
-  data.tar.gz: 17499b71152c831974e0c3d5afb737d3a251e1d5d446775cccfc0179a618a98f49b7aadf570b22fe628f6b83c6b0d87100d71df2f778d47c4b5c1a58145f787d
+  metadata.gz: c1687ba78e8a13d0553dfbba528f36b5cfdfc089289450720ef10890f072b7d7bc31170810fde87825a4a9f0f9bcee0f6f3e5dbfd835f737c5b6edaecdfcf31d
+  data.tar.gz: 8683d2a3cfdaae90039fdb117f0ed8776fcaad8ea919fed0ac2bbaae98ed6c42403f3488d73afa0e73b3285aa4b5628ba99a7d60a98979f5a2379207013f5ce9

data/lib/wayback_machine_downloader.rb

@@ -4,7 +4,7 @@ require_relative 'wayback_machine_downloader/tidy_bytes'
 
 class WaybackMachineDownloader
 
-  VERSION = "0.1.11"
+  VERSION = "0.1.12"
 
   attr_accessor :base_url, :timestamp
 

data/lib/wayback_machine_downloader/tidy_bytes.rb

@@ -0,0 +1,122 @@
+module TibyBytes
+
+  # CP-1252 decimal byte => UTF-8 approximation as an array of bytes
+  CP1252 = {
+    128 => [226, 130, 172],
+    129 => nil,
+    130 => [226, 128, 154],
+    131 => [198, 146],
+    132 => [226, 128, 158],
+    133 => [226, 128, 166],
+    134 => [226, 128, 160],
+    135 => [226, 128, 161],
+    136 => [203, 134],
+    137 => [226, 128, 176],
+    138 => [197, 160],
+    139 => [226, 128, 185],
+    140 => [197, 146],
+    141 => nil,
+    142 => [197, 189],
+    143 => nil,
+    144 => nil,
+    145 => [226, 128, 152],
+    146 => [226, 128, 153],
+    147 => [226, 128, 156],
+    148 => [226, 128, 157],
+    149 => [226, 128, 162],
+    150 => [226, 128, 147],
+    151 => [226, 128, 148],
+    152 => [203, 156],
+    153 => [226, 132, 162],
+    154 => [197, 161],
+    155 => [226, 128, 186],
+    156 => [197, 147],
+    157 => nil,
+    158 => [197, 190],
+    159 => [197, 184]
+  }
+
+  module StringMixin
+
+    # Attempt to replace invalid UTF-8 bytes with valid ones. This method
+    # naively assumes if you have invalid UTF8 bytes, they are either Windows
+    # CP-1252 or ISO8859-1. In practice this isn't a bad assumption, but may not
+    # always work.
+    #
+    # Passing +true+ will forcibly tidy all bytes, assuming that the string's
+    # encoding is CP-1252 or ISO-8859-1.
+    def tidy_bytes(force = false)
+
+      if force
+        return unpack("C*").map do |b|
+          tidy_byte(b)
+        end.flatten.compact.pack("C*").unpack("U*").pack("U*")
+      end
+
+      bytes = unpack("C*")
+      conts_expected = 0
+      last_lead = 0
+
+      bytes.each_index do |i|
+
+        byte          = bytes[i]
+        is_ascii      = byte < 128
+        is_cont       = byte > 127 && byte < 192
+        is_lead       = byte > 191 && byte < 245
+        is_unused     = byte > 240
+        is_restricted = byte > 244
+
+        # Impossible or highly unlikely byte? Clean it.
+        if is_unused || is_restricted
+          bytes[i] = tidy_byte(byte)
+        elsif is_cont
+          # Not expecting contination byte? Clean up. Otherwise, now expect one less.
+          conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1
+        else
+          if conts_expected > 0
+            # Expected continuation, but got ASCII or leading? Clean backwards up to
+            # the leading byte.
+            begin
+              (1..(i - last_lead)).each {|j| bytes[i - j] = tidy_byte(bytes[i - j])}
+            rescue NoMethodError => e
+              next
+            end
+            conts_expected = 0
+          end
+          if is_lead
+            # Final byte is leading? Clean it.
+            if i == bytes.length - 1
+              bytes[i] = tidy_byte(bytes.last)
+            else
+              # Valid leading byte? Expect continuations determined by position of
+              # first zero bit, with max of 3.
+              conts_expected = byte < 224 ? 1 : byte < 240 ? 2 : 3
+              last_lead = i
+            end
+          end
+        end
+      end
+      begin
+        bytes.empty? ? nil : bytes.flatten.compact.pack("C*").unpack("U*").pack("U*")
+      rescue ArgumentError => e
+        nil
+      end
+    end
+
+    # Tidy bytes in-place.
+    def tidy_bytes!(force = false)
+      replace tidy_bytes(force)
+    end
+
+    private
+
+    def tidy_byte(byte)
+      byte < 160 ? TibyBytes::CP1252[byte] : byte < 192 ? [194, byte] : [195, byte - 64]
+    end
+
+  end
+end
+
+class String
+  include TibyBytes::StringMixin
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.12
 platform: ruby
 authors:
 - hartator
@@ -49,6 +49,7 @@ extra_rdoc_files: []
 files:
 - bin/wayback_machine_downloader
 - lib/wayback_machine_downloader.rb
+- lib/wayback_machine_downloader/tidy_bytes.rb
 homepage: https://github.com/hartator/wayback-machine-downloader
 licenses:
 - MIT