percy-cli 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a79b07a50003692a4126836dca74ca72787c479c
-  data.tar.gz: 44b4607a5fe3267aae10e772c7b7057f33ded2eb
+  metadata.gz: 7b25363b82da9f2043e7e5c748bc63d7e7e84535
+  data.tar.gz: f511e267c41c23f9b3cb04263ae6491919a811ce
 SHA512:
-  metadata.gz: 29b2632be605f99a12265a798b7afb40552934bc7445f514a48723ed9842d84672d1524a2f7f1f32fdc1f2615043ed96d14bdb526c3165961a03a0e5590a4549
-  data.tar.gz: 14b7f5941ca563a9bf5c3df841638a162d5c16dd3e3ede57fc644e9fe1190d39ba63dc6e10893f6e6a1e3a644c387e6f652a0bad0f57b0a1035b0b829cd57639
+  metadata.gz: 7b230e067e0ec50a21f27bd6fe0e53152f229a6502f1df9db3e5c50168f0091558d1bf73f5447de4468fef190420c8ee04b8909f7fa3df5161b8a766b4977b6a
+  data.tar.gz: 117c24852676b9870e6baebd3120697db9c1d4ffd609f79819cca57547d7e479cba3a370629897b17f07bf66265eef32b12acd0ee3b74091e90a6e4234ea3d6d

data/README.md

@@ -1,5 +1,8 @@
 # Percy CLI
 
+[![Build Status](https://travis-ci.org/percy/percy-cli.svg?branch=master)](https://travis-ci.org/percy/percy-cli)
+[![Gem Version](https://badge.fury.io/rb/percy-cli.svg)](http://badge.fury.io/rb/percy-cli)
+
 Command-line interface for [Percy](https://percy.io).
 
 ## Installation

data/bin/percy

@@ -3,4 +3,4 @@
 require 'commander'
 require 'percy/cli'
 
-Percy::Cli.new.run if $0 == __FILE__
+Percy::Cli.new.run

data/lib/percy/cli/snapshot.rb

@@ -1,6 +1,9 @@
 require 'find'
 require 'digest'
 require 'uri'
+require 'thread/pool'
+
+Thread::Pool.abort_on_exception = true
 
 module Percy
   class Cli
@@ -44,9 +47,9 @@ module Percy
         # resource path
         "(?:/[^\\s\"']*)?"
       )
-      HTML_REMOTE_URL_REGEX = Regexp.new("<link.*?href=['\"](" + REMOTE_URL_REGEX_STRING + ")")
+      HTML_REMOTE_URL_REGEX = Regexp.new("(<link.*?href=['\"](" + REMOTE_URL_REGEX_STRING + ")[^>]+)")
 
-      # Match all url("https://...") calls, with whitespace and quote variatinos.
+      # Match all url("https://...") styles, with whitespace and quote variatinos.
       CSS_REMOTE_URL_REGEX = Regexp.new(
         "url\\s*\\([\"'\s]*(" + REMOTE_URL_REGEX_STRING + ")[\"'\s]*\\)"
       )
@@ -55,9 +58,8 @@ module Percy
         repo = options[:repo] || Percy.config.repo
         strip_prefix = File.absolute_path(options[:strip_prefix] || root_dir)
         autoload_remote_resources = options[:autoload_remote_resources] || false
-
-        # Create a Percy build for the snapshots.
-        build = Percy.create_build(repo)
+        num_threads = options[:threads] || 10
+        snapshot_limit = options[:snapshot_limit]
 
         # Find all the static files in the given root directory.
         root_paths = find_root_paths(root_dir, snapshots_regex: options[:snapshots_regex])
@@ -70,14 +72,42 @@ module Percy
           related_resources += build_remote_resources(remote_urls)
         end
 
+        all_resources = root_resources + related_resources
+
+        if root_resources.empty?
+          say "No root resource files found. Are there HTML files in the given directory?"
+          exit(-1)
+        end
+
+        say 'Creating build...'
+        build = Percy.create_build(repo, resources: related_resources)
+
+        say 'Uploading build resources...'
+        upload_missing_resources(build, build, all_resources, {num_threads: num_threads})
+
         # Upload a snapshot for every root resource, and associate the related_resources.
+        output_lock = Mutex.new
+        snapshot_thread_pool = Thread.pool(num_threads)
+        total = snapshot_limit ? [root_resources.length, snapshot_limit].min : root_resources.length
         root_resources.each_with_index do |root_resource, i|
-          say "Uploading snapshot (#{i+1}/#{root_resources.length}): #{root_resource.resource_url}"
-          upload_snapshot(build, root_resource, related_resources)
+          break if snapshot_limit && i + 1 > snapshot_limit
+          snapshot_thread_pool.process do
+            output_lock.synchronize do
+              say "Uploading snapshot (#{i+1}/#{total}): #{root_resource.resource_url}"
+            end
+            snapshot = Percy.create_snapshot(build['data']['id'], [root_resource])
+            upload_missing_resources(build, snapshot, all_resources, {num_threads: num_threads})
+            Percy.finalize_snapshot(snapshot['data']['id'])
+          end
         end
+        snapshot_thread_pool.wait
+        snapshot_thread_pool.shutdown
 
         # Finalize the build.
+        say 'Finalizing build...'
         Percy.finalize_build(build['data']['id'])
+        say "Done! Percy is now processing, you can view the visual diffs here:"
+        say build['data']['attributes']['web-url']
       end
 
       private
@@ -121,13 +151,16 @@ module Percy
           case extension
           when '.html'
             content = File.read(path)
-            urls += content.scan(HTML_REMOTE_URL_REGEX).map { |match| maybe_add_protocol(match[0]) }
+            urls += content.scan(HTML_REMOTE_URL_REGEX).map do |match|
+              next if !match[0].include?('stylesheet')  # Only include links with rel="stylesheet".
+              maybe_add_protocol(match[1])
+            end
           when '.css'
             content = File.read(path)
             urls += content.scan(CSS_REMOTE_URL_REGEX).map { |match| maybe_add_protocol(match[0]) }
           end
         end
-        urls.uniq
+        urls.compact.uniq
       end
 
       def maybe_add_protocol(url)
@@ -156,8 +189,8 @@ module Percy
           remote_urls.length,
           title: 'Fetching remote resources...',
           format: ':title |:progress_bar| :percent_complete% complete - :url',
-          width: 40,
-          complete_message: nil,
+          width: 20,
+          complete_message: "Fetched #{remote_urls.length} remote resources.",
         )
 
         remote_urls.each do |url|
@@ -169,7 +202,7 @@ module Percy
             next
           end
           if response.status != 200
-            say_error "Remote resource failed, skipping: #{url}"
+            say_error "Remote resource failed, skipping (#{response.status}): #{url}"
             next
           end
 
@@ -179,34 +212,37 @@ module Percy
         resources
       end
 
-      def upload_snapshot(build, root_resource, related_resources)
-        all_resources = [root_resource] + related_resources
-
-        # Create the snapshot for this page. For simplicity, include all non-HTML resources in the
-        # snapshot as related resources. May seem inefficient, but they will only be uploaded once.
-        snapshot = Percy.create_snapshot(build['data']['id'], all_resources)
-
+      # Uploads missing resources either for a build or snapshot.
+      def upload_missing_resources(build, obj, potential_resources, options = {})
         # Upload the content for any missing resources.
-        missing_resources = snapshot['data']['relationships']['missing-resources']['data']
+        missing_resources = obj['data']['relationships']['missing-resources']['data']
         bar = Commander::UI::ProgressBar.new(
           missing_resources.length,
           title: 'Uploading resources...',
           format: ':title |:progress_bar| :percent_complete% complete - :resource_url',
-          width: 40,
+          width: 20,
           complete_message: nil,
         )
+        output_lock = Mutex.new
+        uploader_thread_pool = Thread.pool(options[:num_threads] || 10)
         missing_resources.each do |missing_resource|
-          missing_resource_sha = missing_resource['id']
-          resource = all_resources.find { |r| r.sha == missing_resource_sha }
-          path = resource.resource_url
-          bar.increment resource_url: resource.resource_url
-
-          # Remote resources are stored in 'content', local resources are read from the filesystem.
-          content = resource.content || File.read("#{resource.path}")
-
-          Percy.upload_resource(build['data']['id'], content)
+          uploader_thread_pool.process do
+            missing_resource_sha = missing_resource['id']
+            resource = potential_resources.find { |r| r.sha == missing_resource_sha }
+            path = resource.resource_url
+            output_lock.synchronize do
+              bar.increment resource_url: resource.resource_url
+            end
+
+            # Remote resources are stored in 'content', local resources are read from the filesystem.
+            content = resource.content || File.read("#{resource.path}")
+
+            Percy.upload_resource(build['data']['id'], content)
+          end
         end
+        uploader_thread_pool.wait
+        uploader_thread_pool.shutdown
       end
     end
   end
-end
+end

data/lib/percy/cli/version.rb

@@ -1,5 +1,5 @@
 module Percy
   class Cli
-    VERSION = '0.0.2'
+    VERSION = '0.1.0'
   end
 end

data/lib/percy/cli.rb

@@ -8,6 +8,9 @@ module Percy
     include Commander::Methods
     include Percy::Cli::Snapshot
 
+    DEFAULT_NUM_THREADS = 10
+    MAX_NUM_THREADS = 50
+
     def say(*args)
       $terminal.say(*args)
     end
@@ -38,13 +41,25 @@ module Percy
           '--snapshots_regex REGEX',
           String,
           'Regular expression for matching the files to snapshot. Defaults to: "\.(html|htm)$"'
+        c.option \
+          '--snapshot_limit NUM',
+          Integer,
+          "Max number of snapshots to upload, useful for testing. Default is unlimited."
         c.option \
           '--autoload_remote_resources',
           'Attempts to parse HTML and CSS for remote resources, fetch them, and include in ' +
           'snapshots. This can be very useful if your static website relies on remote resources.'
+        c.option \
+          '--threads NUM',
+          Integer,
+          "Number of threads in pools for snapshot and resource uploads. " +
+          "Defaults to #{DEFAULT_NUM_THREADS}, max #{MAX_NUM_THREADS}."
 
         c.action do |args, options|
           options.default autoload_remote_resources: false
+          options.default threads: DEFAULT_NUM_THREADS
+          options.threads = MAX_NUM_THREADS if options.threads > MAX_NUM_THREADS
+
           raise OptionParser::MissingArgument, 'root folder path is required' if args.empty?
           if args.length > 1
             raise OptionParser::MissingArgument, 'only a single root folder path can be given'

data/percy-cli.gemspec

@@ -19,8 +19,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'commander', '~> 4'
-  spec.add_dependency 'percy-client', '~> 0.2'
+  spec.add_dependency 'percy-client', '>= 0.2.4'
   spec.add_dependency 'faraday', '>= 0.8'
+  spec.add_dependency 'thread', '~> 0.2'
 
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'

data/spec/percy/cli/testdata/index.html

@@ -12,6 +12,8 @@
 
   <link rel="stylesheet" type="text/css" href="http://example.com:12345/test-diff-tag-order.css">
 
+  <link rel="author" href="https://example.com/should-not-be-included"/>
+
   <h1>Hello World!</h1>
 
   Just some text, should not be included:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: percy-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Perceptual Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-05 00:00:00.000000000 Z
+date: 2015-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander
@@ -28,16 +28,16 @@ dependencies:
   name: percy-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.4
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement