croque 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b91ab14b0b28d3d2a393d4ed507ceed33cd1006
-  data.tar.gz: 4fea7dc1daf1f2a173ebe60bccd1f61e0f2d2490
+  metadata.gz: c4395e1d7c4c85af579598fbcffdb7620700a185
+  data.tar.gz: 0a87659ee43c0475e137a6623065b8ead0338e0a
 SHA512:
-  metadata.gz: b3259b49937d00c49a3396875c3b72cb1627e9b489758548ca3413828c92f6453c8f33746e55d4d7875749bb5b46291a668d48c6a22c176cf2f62008734c9825
-  data.tar.gz: 3ebf0f02950ae93c579f5d3a21842849322fdc47037c12e2448de73961b036b1f0d057b5fb60a7b1d2c4461d16415ea1ec1cce6a0f29739f1a1b5e0f63f94854
+  metadata.gz: f8a3b409feb1f1e6d54521681d541bf1cf8ec5b088122c6add9a08f6fac57552aaed517b1964cce7427cdb01e110d4b299d17f75dfc9c8c94ef02bfbdfba0e0f
+  data.tar.gz: 4e4c5253a5a0638c82bf2eb3db7ca468ee59cec4c973aec544b2678e265fee9440016fbbcc44b38fa055a6b6c2095bfeadd137b5c14510e68c5bae9597fcdb14

data/README.md

@@ -91,6 +91,25 @@ monsieur.query
 => "tomato=delicious&kyouha=hare"
 ```
 
+paginate
+
+```ruby
+Croque.ranking(Date.yesterday. page: 1, per: 50)
+```
+
+all dates whose the ranking exist
+
+```ruby
+Croque.all
+=> [Sat, 21 Oct 2017]
+```
+
+total count of the ranking
+
+```ruby
+Croque.total_count(Date.yesterday)
+=> 1
+```
 
 ## Development
 

data/lib/croque.rb

@@ -41,9 +41,20 @@ module Croque
       Croque::Aggregator.aggregate(date)
     end
 
-    def ranking(date, limit: 0)
+    def all
+      # Get Aggregated List
+      # return date list as Array
+      Croque::Aggregator.all
+    end
+
+    def ranking(date, page: nil, per: nil)
       # Get ranking as Sorted Array
-      Croque::Monsieur.get_list(date, limit)
+      # limit = 0 => all lines
+      Croque::Monsieur.get_list(date, page, per)
+    end
+
+    def total_count(date)
+      Croque::Monsieur.total_count(date)
     end
   end
 end

data/lib/croque/aggregator.rb

@@ -24,7 +24,9 @@ module Croque
           while (k-1)*linage < line_count.to_i
             fragment = `head -n #{k*1000} #{file} | tail -n #{linage}`
             fragment_lines = fragment.lines
-            lines += fragment_lines.select{ |line| line.match(date_matcher(date)) }
+            lines += fragment_lines.select do |line|
+              line.match(date_matcher(date))
+            end
             k += 1
           end
           # extract the matched line (Date)
@@ -62,6 +64,16 @@ module Croque
         store_csv(ranking_path(date), data)
       end
 
+      def all
+        paths = Dir.glob(store_path + '*')
+        paths = paths.select do |path|
+          path.match(/\d{4}\-\d{2}\-\d{2}/)
+        end
+        paths.map do |path|
+          Date.parse(File.basename(path))
+        end
+      end
+
       private
       def log_files
         Dir::glob(dir_path + '*').select do |path|
@@ -73,8 +85,12 @@ module Croque
         Croque.config.log_dir_path
       end
 
+      def store_path
+        Croque.config.store_path
+      end
+
       def ranking_path(date)
-        Croque.config.store_path.join("#{date}", "ranking.csv")
+        store_path.join("#{date}", "ranking.csv")
       end
 
       def log_file_matcher
@@ -82,7 +98,7 @@ module Croque
       end
 
       def remove_files(date)
-        path = Croque.config.store_path.join("#{date}")
+        path = store_path.join("#{date}")
         if Dir.exist?(path)
           FileUtils.remove_dir(path)
         end
@@ -92,7 +108,7 @@ module Croque
         # matcher
         matcher = convert_matcher(matcher: Croque.config.matcher)
         # head
-        head_lines = `head -n 10 #{file}`
+        head_lines = `head -n 100 #{file}`
         # get lines as Array
         head_lines = head_lines.lines
         head_line = head_lines.select do |line|
@@ -100,7 +116,7 @@ module Croque
         end.first
         head_date = get_date_from_line(head_line)
         # tail
-        tail_lines = `tail -n 10 #{file}`
+        tail_lines = `tail -n 100 #{file}`
         # get lines as Array
         tail_lines = tail_lines.lines
         tail_line = tail_lines.select do |line|
@@ -190,7 +206,7 @@ module Croque
       end
 
       def csv_path(date, hour)
-        Croque.config.store_path.join("#{date}", "#{hour}.csv")
+        store_path.join("#{date}", "#{hour}.csv")
       end
 
       def csv_option

data/lib/croque/monsieur.rb

@@ -46,16 +46,31 @@ module Croque
     end
 
     class << self
-      def get_list(date, limit)
+      def get_list(date, page, per)
+        # maybe String
+        page_num = (page || 1).to_i
+        per_num = (per || total_count(date)).to_i
+        # get csv data
         csv_data = File.open(ranking_path(date), "r").read.gsub(/\r/, "")
         csv = CSV.new(csv_data)
         # Sorted lines as ranking
-        csv.to_a[0..(limit-1)].map do |line|
+        start = ((page_num-1)*per_num)
+        # csv to Array
+        lines = csv.to_a
+        # start..end
+        lines = lines.slice(start, per_num) || []
+        # generate this class instance
+        lines.map do |line|
           # line = [date, hour, uuid, processing_time (ms)]
           self.new(*line)
         end
       end
 
+      def total_count(date)
+        wc_result = `wc -l #{ranking_path(date)}`
+        wc_result.match(/\d+/)[0].try(:to_i)
+      end
+
       private
       def ranking_path(date)
         Croque.config.store_path.join("#{date}", "ranking.csv")

data/lib/croque/version.rb

@@ -1,3 +1,3 @@
 module Croque
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/log/development.log

@@ -1,4 +1,8 @@
 # Logfile created on 2017-10-21 12:52:41 +0900 by logger.rb/56815
+I, [2017-10-21T06:55:28.313141 #20032]  INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:28 +0900
+I, [2017-10-21T06:55:28.112109 #20032]  INFO -- : Processing by Rails::WelcomeController#index as HTML
+I, [2017-10-21T06:55:28.898918 #20032]  INFO -- : Completed 200 OK in 1111ms (Views: 1111.0ms)
+
 I, [2017-10-21T12:53:04.553805 #11178]  INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:53:04 +0900
 I, [2017-10-21T12:53:04.566846 #11178]  INFO -- : Processing by Rails::WelcomeController#index as HTML
 I, [2017-10-21T12:55:04.566846 #22212]  INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900

data/log/test.log

@@ -1,4 +1,8 @@
 # Logfile created on 2017-10-21 12:52:41 +0900 by logger.rb/56815
+I, [2017-10-21T06:55:28.313141 #20032]  INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:28 +0900
+I, [2017-10-21T06:55:28.112109 #20032]  INFO -- : Processing by Rails::WelcomeController#index as HTML
+I, [2017-10-21T06:55:28.898918 #20032]  INFO -- : Completed 200 OK in 1111ms (Views: 1111.0ms)
+
 I, [2017-10-21T12:53:04.553805 #11178]  INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:53:04 +0900
 I, [2017-10-21T12:53:04.566846 #11178]  INFO -- : Processing by Rails::WelcomeController#index as HTML
 I, [2017-10-21T12:55:04.566846 #22212]  INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: croque
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Takuya Okuhara
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-21 00:00:00.000000000 Z
+date: 2017-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails