bot_twitter_ebooks 3.2.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9bd28ebd17798bf3a09edbd2ecac05e840d3be98
-  data.tar.gz: 035fcd79ba4d452030d514b5389792c11631a84d
+  metadata.gz: 0a68a4d52f738569df003bfac626ffeff9ef8a27
+  data.tar.gz: 0f639a347f2d15988d1559bcef4c09a78b66cbd5
 SHA512:
-  metadata.gz: 42440202f4e9fb012ed01261cc575107e82ac2e21450241c3d12d912ff9921c17d47be37abe8cc0aa504cd64d5c858aaac78ace66a8f28602f796a493216b46d
-  data.tar.gz: c1872982f4d8c16487fbfe70bcecf71b560105c052c96f06237a4bd3acf35028adc4c4ab9a15c27eacb5f19fd9628a1adadb81528949c71da17ce0488b7b051b
+  metadata.gz: 4ca1cc6ceed9c9a71313d92b9f844eb923382f9f28d38766d3fe3e8772420e05e41312a8392fd7ee15f5a9dba89288d92da926c0079d0eee7fba78d158b9f92d
+  data.tar.gz: 7fe92c90844b8b4e34a4dd75df136006b994925c40f998ac1f2643dd181dfbbc01ba14c64c19e1baaa56c167f4a1f40e8b404a4fd3bba8166a72c9731830d4d2

data/.gitignore

@@ -1,5 +1,250 @@
 
-# Created by https://www.gitignore.io/api/git,vim,ruby,linux,macos,emacs,windows
+# Created by https://www.gitignore.io/api/git,vim,linux,macos,emacs,ruby,dropbox,windows,archive,vagrant,archives,compressed,compression,visualstudiocode,archlinuxpackages,compressedarchive
+
+### Archive ###
+
+### Mostly from https://en.wikipedia.org/wiki/List_of_archive_formats
+
+## Archiving only
+# The traditional archive format on Unix-like systems, now used mainly for the creation of static libraries.
+*.a
+*.ar
+# RPM files consist of metadata concatenated with (usually) a cpio archive. Newer RPM systems also support other archives, as cpio is becoming obsolete. cpio is also used with initramfs.
+*.cpio
+
+# A self-extracting archive that uses the Bourne shell (sh).
+*.shar
+# A system for storing multiple files. LBR archives typically contained files processed by SQ, or the archive itself was compressed with SQ. LBR archives that were compressed with SQ ended with the extension .LQR
+*.LBR
+# An archive format originally used mainly for archiving and distribution of the exact, nearly-exact, or custom-modified contents of an optical storage medium such as a CD-ROM or DVD-ROM. However, it can be used to archive the contents of other storage media, selected partitions, folders, and/or files. The resulting archive is typically optimized for convenient rendering to (re-)writable CD or DVD media.
+*.iso
+# A library format used primarily on the Commodore 64 and 128 lines of computers. This bears no resemblance to the DOS LBR format. While library files were quick to implement (a number of programs exist to work with them) they are crippled in that they cannot grow with use: once a file has been created it cannot be amended (files added, changed or deleted) without recreating the entire file.
+*.lbr
+# An archive format used by Mozilla for storing binary diffs. Used in conjunction with bzip2.
+*.mar
+# A common archive format used on Unix-like systems. Generally used in conjunction with compressors such as gzip, bzip2, compress or xz to create .tar.gz, .tar.bz2, .tar.Z or tar.xz files.
+*.tar
+
+# Package managers
+# Red Hat Package Manager
+*.rpm
+# Debian package
+*.deb
+# MicroSoft Installer
+*.msi
+*.msm
+*.msp
+# Mozilla package installer
+*.xpi
+# Ruby Package
+*.gem
+
+
+### Archives ###
+# It's better to unpack these files and commit the raw source because
+# git has its own built in compression methods.
+*.7z
+*.jar
+*.rar
+*.zip
+*.gz
+*.tgz
+*.bzip
+*.bz2
+*.xz
+*.lzma
+*.cab
+
+#packing-only formats
+
+#package management formats
+*.dmg
+*.egg
+
+### ArchLinuxPackages ###
+*.tar.*
+*.exe
+*.log
+*.log.*
+*.sig
+
+pkg/
+src/
+
+### Compressed ###
+*.pkg
+*.sit
+*.sitx
+*.zipx
+
+### CompressedArchive ###
+
+### Mostly from https://en.wikipedia.org/wiki/List_of_archive_formats
+
+## Archiving and compression
+# Open source file format. Used by 7-Zip.
+# Mac OS X, restoration on different platforms is possible although not immediate 	Yes 	Based on 7z. Preserves Spotlight metadata, resource forks, owner/group information, dates and other data which would be otherwise lost with compression.
+*.s7z
+# Old archive versions only 	Proprietary format
+*.ace
+# A format that compresses and doubly encrypt the data (AES256 and CAS256) avoiding brute force attacks, also hide files in an AFA file. It has two ways to safeguard data integrity and subsequent repair of the file if has an error (repair with AstroA2P (online) or Astrotite (offline)).
+*.afa
+# A mainly Korean format designed for very large archives.
+*.alz
+# Android application package (variant of JAR file format).
+*.apk
+# ??
+*.arc
+# Originally DOS, now multiple
+*.arj
+# Open archive format, used by B1 Free Archiver (http://dev.b1.org/standard/archive-format.html)
+*.b1
+# Binary Archive with external header
+*.ba
+# Proprietary format from the ZipTV Compression Components
+*.bh
+# The Microsoft Windows native archive format, which is also used by many commercial installers such as InstallShield and WISE.
+# Originally DOS, now DOS and Windows 	Created by Yaakov Gringeler; released last in 2003 (Compressia 1.0.0.1 beta), now apparently defunct. Free trial of 30 days lets user create and extract archives; after that it is possible to extract, but not to create.
+*.car
+# Open source file format.
+*.cfs
+# Compact Pro archive, a common archiver used on Mac platforms until about Mac OS 7.5.x. Competed with StuffIt; now obsolete.
+*.cpt
+# Windows, Unix-like, Mac OS X Open source file format. Files are compressed individually with either gzip, bzip2 or lzo.
+*.dar
+# DiskDoubler 	Mac OS 			obsolete
+*.dd
+# ??
+*.dgc
+# Apple Disk Image upports "Internet-enabled" disk images, which, once downloaded, are automatically decompressed, mounted, have the contents extracted, and thrown away. Currently, Safari is the only browser that supports this form of extraction; however, the images can be manually extracted as well. This format can also be password-protected or encrypted with 128-bit or 256-bit AES encryption.
+# Enterprise Java Archive archive
+*.ear
+# ETSoft compressed archive
+# The predecessor of DGCA.
+*.gca
+# Originally DOS 	Yes, but may be covered by patents 	DOS era format; uses arithmetic/Markov coding
+*.ha
+# MS Windows 	HKI
+*.hki
+# Produced by ICEOWS program. Excels at text file compression.
+*.ice
+# Java archive, compatible with ZIP files
+*.jar
+# Open sourced archiver with compression using the PAQ family of algorithms and optional encryption.
+*.kgb
+# Originally DOS, now multiple 	Multiple 	Yes 	The standard format on Amiga.
+*.lzh
+*.lha
+# Archiver originally used on The Amiga. Now copied by Microsoft to use in their .cab and .chm files.
+*.lzx
+# file format from NoGate Consultings, a rival from ARC-Compressor.
+*.pak
+# A disk image archive format that supports several compression methods as well as splitting the archive into smaller pieces.
+*.partimg
+# An experimental open source packager (http://mattmahoney.net/dc)
+*.paq*
+# Open source archiver supporting authenticated encryption, volume spanning, customizable object level and volume level integrity checks (form CRCs to SHA-512 and Whirlpool hashes), fast deflate based compression
+*.pea
+# The format from the PIM - a freeware compression tool by Ilia Muraviev. It uses an LZP-based compression algorithm with set of filters for executable, image and audio files.
+*.pim
+# PackIt 	Mac OS 			obsolete
+*.pit
+# Used for data in games written using the Quadruple D library for Delphi. Uses byte pair compression.
+*.qda
+# A proprietary archive format, second in popularity to .zip files.
+# The format from a commercial archiving package. Odd among commercial packages in that they focus on incorporating experimental algorithms with the highest possible compression (at the expense of speed and memory), such as PAQ, PPMD and PPMZ (PPMD with unlimited-length strings), as well as a proprietary algorithms.
+*.rk
+# Self Dissolving ARChive 	Commodore 64, Commodore 128 	Commodore 64, Commodore 128 	Yes 	SDAs refer to Self Dissolving ARC files, and are based on the Commodore 64 and Commodore 128 versions of ARC, originally written by Chris Smeets. While the files share the same extension, they are not compatible between platforms. That is, an SDA created on a Commodore 64 but run on a Commodore 128 in Commodore 128 mode will crash the machine, and vice versa. The intended successor to SDA is SFX.
+*.sda
+# A pre-Mac OS X Self-Extracting Archive format. StuffIt, Compact Pro, Disk Doubler and others could create .sea files, though the StuffIt versions were the most common.
+*.sea
+# Scifer Archive with internal header
+*.sen
+# Commodore 64, Commodore 128 	SFX is a Self Extracting Archive which uses the LHArc compression algorithm. It was originally developed by Chris Smeets on the Commodore platform, and runs primarily using the CS-DOS extension for the Commodore 128. Unlike its predecessor SDA, SFX files will run on both the Commodore 64 and Commodore 128 regardless of which machine they were created on.
+*.sfx
+# An archive format designed for the Apple II series of computers. The canonical implementation is ShrinkIt, which can operate on disk images as well as files. Preferred compression algorithm is a combination of RLE and 12-bit LZW. Archives can be manipulated with the command-line NuLib tool, or the Windows-based CiderPress.
+*.shk
+# A compression format common on Apple Macintosh computers. The free StuffIt Expander is available for Windows and OS X.
+# The replacement for the .sit format that supports more compression methods, UNIX file permissions, long file names, very large files, more encryption options, data specific compressors (JPEG, Zip, PDF, 24-bit image, MP3). The free StuffIt Expander is available for Windows and OS X.
+# A royalty-free compressing format
+*.sqx
+# The "tarball" format combines tar archives with a file-based compression scheme (usually gzip). Commonly used for source and binary distribution on Unix-like platforms, widely available elsewhere.
+*.tar.gz
+*.tar.Z
+*.tar.bz2
+*.tbz2
+*.tar.lzma
+*.tlz
+# UltraCompressor 2.3 was developed to act as an alternative to the then popular PKZIP application. The main feature of the application is its ability to create large archives. This means that compressed archives with the UC2 file extension can hold almost 1 million files.
+*.uc
+*.uc0
+*.uc2
+*.ucn
+*.ur2
+*.ue2
+# Based on PAQ, RZM, CSC, CCM, and 7zip. The format consists of a PAQ, RZM, CSC, or CCM compressed file and a manifest with compression settings stored in a 7z archive.
+*.uca
+# A high compression rate archive format originally for DOS.
+*.uha
+# Web Application archive (Java-based web app)
+*.war
+# File-based disk image format developed to deploy Microsoft Windows.
+*.wim
+# XAR
+*.xar
+# Native format of the Open Source KiriKiri Visual Novel engine. Uses combination of block splitting and zlib compression. The filenames and pathes are stored in UTF-16 format. For integrity check, the Adler-32 hashsum is used. For many commercial games, the files are encrypted (and decoded on runtime) via so-called "cxdec" module, which implements xor-based encryption.
+*.xp3
+# Yamazaki zipper archive. Compression format used in DeepFreezer archiver utility created by Yamazaki Satoshi. Read and write support exists in TUGZip, IZArc and ZipZag
+*.yz1
+# The most widely used compression format on Microsoft Windows. Commonly used on Macintosh and Unix systems as well.
+# application/x-zoo 	zoo 	Multiple 	Multiple 	Yes
+*.zoo
+# Journaling (append-only) archive format with rollback capability. Supports deduplication and incremental update based on last-modified dates. Multi-threaded. Compresses in LZ77, BWT, and context mixing formats. Open source.
+*.zpaq
+# Archiver with a compression algorithm based on the Burrows-Wheeler transform method.
+*.zz
+
+
+### Compression ###
+
+### From https://en.wikipedia.org/wiki/List_of_archive_formats
+
+## Compression only
+# An open source, patent- and royalty-free compression format. The compression algorithm is a Burrows-Wheeler transform followed by a move-to-front transform and finally Huffman coding
+# Old compressor for QNX4 OS. The compression algorithm is a modified LZSS, with an adaptive Huffman coding.
+*.F
+# GNU Zip, the primary compression format used by Unix-like systems. The compression algorithm is DEFLATE.
+# An alternate LZMA algorithm implementation, with support for checksums and ident bytes.
+*.lz
+# The LZMA compression algorithm as used by 7-Zip
+# An implementation of the LZO data compression algorithm
+*.lzo
+# A compression program designed to do particularly well on very large files containing long distance redundancy.
+*.rz
+# Windows compress/decompress- Linux and Mac OS X decompress only 	A compression program designed to do high compression on SF2 files (SoundFont)
+*.sfark
+# A compression format invented by Google and open-sourced in 2011. Snappy aims for very high speeds, reasonable compression, and maximum stability rather than maximum compression or compatibility with any other compression library.
+*.sz
+# Squeeze: A program which compressed files. A file which was "squeezed" had the middle initial of the name changed to "Q", so that a squeezed text file would end with .TQT, a squeezed executable would end with .CQM or .EQE. Typically used with .LBR archives, either by storing the squeezed files in the archive, or by storing the files decompressed and then compressing the archive, which would have a name ending in ".LQR".
+*.?Q?
+# A compression program written by Steven Greenberg implementing the LZW algorithm. For several years in the CP/M world when no implementation was available of ARC, CRUNCHed files stored in .LBR archives were very popular. CRUNCH's implementation of LZW had a somewhat unique feature of modifying and occasionally clearing the code table in memory when it became full, resulting in a few percent better compression on many files.
+*.?Z?
+# A compression format using LZMA2 to yield very high compression ratios.
+# The traditional Huffman coding compression format.
+*.z
+# The traditional LZW compression format.
+*.Z
+# Joke compression program, actually increasing file size
+*.infl
+# Compression format(s) used by some DOS and Windows install programs. MS-DOS includes expand.exe to decompress its install files. The compressed files are created with a matching compress.exe command. The compression algorithm is LZSS.
+*.??_
+
+
+### Dropbox ###
+# Dropbox settings and caches
+.dropbox
+.dropbox.attr
+.dropbox.cache
 
 ### Emacs ###
 # -*- mode: gitignore; -*-
@@ -112,7 +357,6 @@ Temporary Items
 .apdisk
 
 ### Ruby ###
-*.gem
 *.rbc
 /.config
 /coverage/
@@ -163,6 +407,9 @@ Gemfile.lock
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
 
+### Vagrant ###
+.vagrant/
+
 ### Vim ###
 # swap
 [._]*.s[a-v][a-z]
@@ -176,6 +423,14 @@ Session.vim
 # auto-generated tag files
 tags
 
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+.history
+
 ### Windows ###
 # Windows thumbnail cache files
 Thumbs.db
@@ -189,12 +444,8 @@ Desktop.ini
 $RECYCLE.BIN/
 
 # Windows Installer files
-*.cab
-*.msi
-*.msm
-*.msp
 
 # Windows shortcuts
 *.lnk
 
-# End of https://www.gitignore.io/api/git,vim,ruby,linux,macos,emacs,windows
+# End of https://www.gitignore.io/api/git,vim,linux,macos,emacs,ruby,dropbox,windows,archive,vagrant,archives,compressed,compression,visualstudiocode,archlinuxpackages,compressedarchive

data/README.md

@@ -21,7 +21,7 @@ Note that 3.0 is not backwards compatible with 2.x, so upgrade carefully! In par
 
 ## Installation
 
-Requires Ruby 2.1+. Ruby 2.4+ is recommended.
+Requires Ruby 2.4+.
 
 ```bash
 gem install bot_twitter_ebooks

data/bot_twitter_ebooks.gemspec

@@ -2,7 +2,7 @@
 require File.expand_path('../lib/bot_twitter_ebooks/version', __FILE__)
 
 Gem::Specification.new do |spec|
-  spec.required_ruby_version = '>= 2.1'
+  spec.required_ruby_version = '>= 2.4'
 
   spec.name          = "bot_twitter_ebooks"
   spec.version       = Ebooks::VERSION

data/lib/bot_twitter_ebooks/model.rb

@@ -127,7 +127,7 @@ module Ebooks
       sentences.map do |s|
         tokens = NLP.tokenize(s).reject do |t|
           # Don't include usernames/urls as tokens
-          t.include?('@') || t.include?('http')
+          t.start_with?('@') || t.downcase.start_with?('http')
         end
 
         tokens.map { |t| tikify(t) }
@@ -169,7 +169,7 @@ module Ebooks
       mentions = []
       lines.each do |l|
         next if l.start_with?('#') # Remove commented lines
-        next if l.include?('RT') || l.include?('MT') # Remove soft retweets
+        next if l.include?(': "RT @') || l.include?(': "MT @') # Remove soft retweets
 
         if l.include?('@')
           mentions << NLP.normalize(l)
@@ -190,7 +190,7 @@ module Ebooks
 
       log "Ranking keywords"
       @keywords = NLP.keywords(text).top(200).map(&:to_s)
-      log "Top keywords: #{@keywords[0]} #{@keywords[1]} #{@keywords[2]}"
+      log "Top keywords: #{@keywords[0]} #{@keywords[1]} #{@keywords[2]} #{@keywords[3]} #{@keywords[4]} #{@keywords[5]} #{@keywords[6]} #{@keywords[7]}"
 
       self
     end

data/lib/bot_twitter_ebooks/nlp.rb

@@ -7,7 +7,7 @@ module Ebooks
   module NLP
     # We deliberately limit our punctuation handling to stuff we can do consistently
     # It'll just be a part of another token if we don't split it out, and that's fine
-    PUNCTUATION = ".?!,"
+    PUNCTUATION = ".¿?¡!,"
 
     # Lazy-load NLP libraries and resources
     # Some of this stuff is pretty heavy and we don't necessarily need
@@ -86,10 +86,12 @@ module Ebooks
     # @param text [String]
     # @return [Highscore::Keywords]
     def self.keywords(text)
-      # Preprocess to remove stopwords (highscore's blacklist is v. slow)
-      text = NLP.tokenize(text).reject { |t| stopword?(t) }.join(' ')
+      # Preprocess to remove stopwords and urls (highscore's blacklist is v. slow)
+      text = NLP.tokenize(text).reject do |t|
+        t.downcase.start_with?('http') || stopword?(t)
+      end
 
-      text = Highscore::Content.new(text)
+      text = Highscore::Content.new(text.join(' '))
 
       text.configure do
         #set :multiplier, 2
@@ -98,7 +100,7 @@ module Ebooks
         #set :long_words_threshold, 15
         #set :vowels, 1                     # => default: 0 = not considered
         #set :consonants, 5                 # => default: 0 = not considered
-        #set :ignore_case, true             # => default: false
+        set :ignore_case, true             # => default: false
         set :word_pattern, /(?<!@)(?<=\s)[\p{Word}']+/           # => default: /\w+/
         #set :stemming, true                # => default: false
       end

data/lib/bot_twitter_ebooks/version.rb

@@ -1,3 +1,3 @@
 module Ebooks
-  VERSION = "3.2.0"
+  VERSION = "3.3.0"
 end

data/spec/model_spec.rb

@@ -39,8 +39,8 @@ describe Ebooks::Model do
     expect(report2.total_memsize).to be < 4000000
 
     expect(model.tokens[0]).to be_a String
-    expect(model.sentences[0][0]).to be_a Fixnum
-    expect(model.mentions[0][0]).to be_a Fixnum
+    expect(model.sentences[0][0]).to be_a Integer
+    expect(model.mentions[0][0]).to be_a Integer
     expect(model.keywords[0]).to be_a String
 
     puts "elonmusk.model uses #{report2.total_memsize} bytes in memory"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bot_twitter_ebooks
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - astrolince
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-29 00:00:00.000000000 Z
+date: 2017-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -289,7 +289,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="