nanoc 4.1.6 → 4.2.0b1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -0
  3. data/Gemfile.lock +2 -1
  4. data/NEWS.md +11 -4
  5. data/lib/nanoc/base/checksummer.rb +135 -46
  6. data/lib/nanoc/base/compilation/compiler.rb +18 -28
  7. data/lib/nanoc/base/compilation/dependency_tracker.rb +22 -32
  8. data/lib/nanoc/base/compilation/filter.rb +2 -4
  9. data/lib/nanoc/base/entities.rb +1 -0
  10. data/lib/nanoc/base/entities/content.rb +14 -3
  11. data/lib/nanoc/base/entities/document.rb +14 -6
  12. data/lib/nanoc/base/entities/item.rb +0 -31
  13. data/lib/nanoc/base/entities/item_rep.rb +1 -1
  14. data/lib/nanoc/base/entities/lazy_value.rb +36 -0
  15. data/lib/nanoc/base/entities/pattern.rb +3 -2
  16. data/lib/nanoc/base/entities/site.rb +2 -0
  17. data/lib/nanoc/base/memoization.rb +17 -10
  18. data/lib/nanoc/base/repos/compiled_content_cache.rb +1 -1
  19. data/lib/nanoc/base/repos/data_source.rb +10 -6
  20. data/lib/nanoc/base/services/executor.rb +22 -22
  21. data/lib/nanoc/base/services/item_rep_router.rb +4 -5
  22. data/lib/nanoc/base/views.rb +0 -1
  23. data/lib/nanoc/base/views/item_rep_view.rb +3 -9
  24. data/lib/nanoc/base/views/mixins/document_view_mixin.rb +4 -11
  25. data/lib/nanoc/base/views/view.rb +1 -0
  26. data/lib/nanoc/base/views/view_context.rb +5 -1
  27. data/lib/nanoc/cli/commands/compile.rb +0 -6
  28. data/lib/nanoc/data_sources.rb +5 -5
  29. data/lib/nanoc/data_sources/filesystem.rb +219 -90
  30. data/lib/nanoc/extra/checking/check.rb +1 -2
  31. data/lib/nanoc/extra/checking/checks.rb +2 -0
  32. data/lib/nanoc/extra/checking/checks/css.rb +6 -14
  33. data/lib/nanoc/extra/checking/checks/html.rb +6 -14
  34. data/lib/nanoc/extra/checking/checks/internal_links.rb +14 -3
  35. data/lib/nanoc/extra/checking/checks/w3c_validator.rb +28 -0
  36. data/lib/nanoc/extra/deployers/fog.rb +134 -78
  37. data/lib/nanoc/extra/link_collector.rb +14 -18
  38. data/lib/nanoc/filters/sass.rb +3 -3
  39. data/lib/nanoc/helpers.rb +1 -0
  40. data/lib/nanoc/helpers/capturing.rb +16 -58
  41. data/lib/nanoc/helpers/child_parent.rb +51 -0
  42. data/lib/nanoc/helpers/filtering.rb +0 -1
  43. data/lib/nanoc/helpers/html_escape.rb +5 -0
  44. data/lib/nanoc/helpers/link_to.rb +2 -0
  45. data/lib/nanoc/helpers/rendering.rb +3 -4
  46. data/lib/nanoc/rule_dsl/action_provider.rb +20 -4
  47. data/lib/nanoc/rule_dsl/recording_executor.rb +3 -1
  48. data/lib/nanoc/rule_dsl/rule_context.rb +0 -1
  49. data/lib/nanoc/rule_dsl/rule_memory_calculator.rb +4 -1
  50. data/lib/nanoc/spec.rb +217 -0
  51. data/lib/nanoc/version.rb +1 -1
  52. data/test/base/test_data_source.rb +4 -2
  53. data/test/base/test_dependency_tracker.rb +5 -11
  54. data/test/data_sources/test_filesystem.rb +605 -69
  55. data/test/extra/checking/checks/test_internal_links.rb +25 -0
  56. data/test/extra/deployers/test_fog.rb +0 -177
  57. data/test/filters/test_less.rb +9 -4
  58. data/test/helpers/test_capturing.rb +38 -212
  59. data/test/helpers/test_link_to.rb +0 -205
  60. data/test/helpers/test_xml_sitemap.rb +2 -1
  61. metadata +7 -12
  62. data/lib/nanoc/base/views/site_view.rb +0 -14
  63. data/lib/nanoc/data_sources/filesystem_unified.rb +0 -101
  64. data/test/data_sources/test_filesystem_unified.rb +0 -559
  65. data/test/helpers/test_breadcrumbs.rb +0 -60
  66. data/test/helpers/test_filtering.rb +0 -112
  67. data/test/helpers/test_html_escape.rb +0 -26
  68. data/test/helpers/test_rendering.rb +0 -147
  69. data/test/helpers/test_tagging.rb +0 -92
  70. data/test/helpers/test_text.rb +0 -18
@@ -20,13 +20,12 @@ module Nanoc::Extra::Checking
20
20
  output_filenames = Dir[output_dir + '/**/*'].select { |f| File.file?(f) }
21
21
 
22
22
  # FIXME: ugly
23
- view_context = site.compiler.create_view_context
23
+ view_context = site.compiler.create_view_context(Nanoc::Int::DependencyTracker::Null.new)
24
24
 
25
25
  context = {
26
26
  items: Nanoc::ItemCollectionWithRepsView.new(site.items, view_context),
27
27
  layouts: Nanoc::LayoutCollectionView.new(site.layouts, view_context),
28
28
  config: Nanoc::ConfigView.new(site.config, view_context),
29
- site: Nanoc::SiteView.new(site, view_context), # TODO: remove me
30
29
  output_filenames: output_filenames,
31
30
  }
32
31
 
@@ -1,3 +1,5 @@
1
+ require_relative 'checks/w3c_validator'
2
+
1
3
  # @api private
2
4
  module Nanoc::Extra::Checking::Checks
3
5
  autoload 'CSS', 'nanoc/extra/checking/checks/css'
@@ -1,22 +1,14 @@
1
1
  module ::Nanoc::Extra::Checking::Checks
2
2
  # @api private
3
- class CSS < ::Nanoc::Extra::Checking::Check
3
+ class CSS < ::Nanoc::Extra::Checking::Checks::W3CValidator
4
4
  identifier :css
5
5
 
6
- def run
7
- require 'w3c_validators'
6
+ def extension
7
+ 'css'
8
+ end
8
9
 
9
- Dir[@config[:output_dir] + '/**/*.css'].each do |filename|
10
- results = ::W3CValidators::CSSValidator.new.validate_file(filename)
11
- lines = File.readlines(filename)
12
- results.errors.each do |e|
13
- line_num = e.line.to_i - 1
14
- line = lines[line_num]
15
- message = e.message.gsub(%r{\s+}, ' ').strip.sub(/\s+:$/, '')
16
- desc = "line #{line_num + 1}: #{message}: #{line}"
17
- add_issue(desc, subject: filename)
18
- end
19
- end
10
+ def validator_class
11
+ ::W3CValidators::CSSValidator
20
12
  end
21
13
  end
22
14
  end
@@ -1,22 +1,14 @@
1
1
  module ::Nanoc::Extra::Checking::Checks
2
2
  # @api private
3
- class HTML < ::Nanoc::Extra::Checking::Check
3
+ class HTML < ::Nanoc::Extra::Checking::Checks::W3CValidator
4
4
  identifier :html
5
5
 
6
- def run
7
- require 'w3c_validators'
6
+ def extension
7
+ '{htm,html}'
8
+ end
8
9
 
9
- Dir[@config[:output_dir] + '/**/*.{htm,html}'].each do |filename|
10
- results = ::W3CValidators::MarkupValidator.new.validate_file(filename)
11
- lines = File.readlines(filename)
12
- results.errors.each do |e|
13
- line_num = e.line.to_i - 1
14
- line = lines[line_num]
15
- message = e.message.gsub(%r{\s+}, ' ').strip.sub(/\s+:$/, '')
16
- desc = "line #{line_num + 1}: #{message}: #{line}"
17
- add_issue(desc, subject: filename)
18
- end
19
- end
10
+ def validator_class
11
+ ::W3CValidators::MarkupValidator
20
12
  end
21
13
  end
22
14
  end
@@ -34,7 +34,7 @@ module Nanoc::Extra::Checking::Checks
34
34
  return true if href == '.'
35
35
 
36
36
  # Skip hrefs that are specified in the exclude configuration
37
- return true if excluded?(href)
37
+ return true if excluded?(href, origin)
38
38
 
39
39
  # Remove target
40
40
  path = href.sub(/#.*$/, '')
@@ -65,9 +65,20 @@ module Nanoc::Extra::Checking::Checks
65
65
  false
66
66
  end
67
67
 
68
- def excluded?(href)
69
- excludes = @config.fetch(:checks, {}).fetch(:internal_links, {}).fetch(:exclude, [])
68
+ def excluded?(href, origin)
69
+ config = @config.fetch(:checks, {}).fetch(:internal_links, {})
70
+ excluded_target?(href, config) || excluded_origin?(origin, config)
71
+ end
72
+
73
+ def excluded_target?(href, config)
74
+ excludes = config.fetch(:exclude_targets, config.fetch(:exclude, []))
70
75
  excludes.any? { |pattern| Regexp.new(pattern).match(href) }
71
76
  end
77
+
78
+ def excluded_origin?(origin, config)
79
+ relative_origin = origin[@config[:output_dir].size..-1]
80
+ excludes = config.fetch(:exclude_origins, [])
81
+ excludes.any? { |pattern| Regexp.new(pattern).match(relative_origin) }
82
+ end
72
83
  end
73
84
  end
@@ -0,0 +1,28 @@
1
+ module ::Nanoc::Extra::Checking::Checks
2
+ # @api private
3
+ class W3CValidator < ::Nanoc::Extra::Checking::Check
4
+ def run
5
+ require 'w3c_validators'
6
+
7
+ Dir[@config[:output_dir] + '/**/*.' + extension].each do |filename|
8
+ results = validator_class.new.validate_file(filename)
9
+ lines = File.readlines(filename)
10
+ results.errors.each do |e|
11
+ line_num = e.line.to_i - 1
12
+ line = lines[line_num]
13
+ message = e.message.gsub(%r{\s+}, ' ').strip.sub(/\s+:$/, '')
14
+ desc = "line #{line_num + 1}: #{message}: #{line}"
15
+ add_issue(desc, subject: filename)
16
+ end
17
+ end
18
+ end
19
+
20
+ def extension
21
+ raise NotImplementedError
22
+ end
23
+
24
+ def validator_class
25
+ raise NotImplementedError
26
+ end
27
+ end
28
+ end
@@ -21,104 +21,165 @@ module Nanoc::Extra::Deployers
21
21
  #
22
22
  # @api private
23
23
  class Fog < ::Nanoc::Extra::Deployer
24
+ class FogWrapper
25
+ def initialize(directory, is_dry_run)
26
+ @directory = directory
27
+ @is_dry_run = is_dry_run
28
+ end
29
+
30
+ def upload(source_filename, destination_key)
31
+ log_effectful("uploading #{source_filename} -> #{destination_key}")
32
+
33
+ unless dry_run?
34
+ # FIXME: source_filename file is never closed
35
+ @directory.files.create(
36
+ key: destination_key,
37
+ body: File.open(source_filename),
38
+ public: true,
39
+ )
40
+ end
41
+ end
42
+
43
+ def remove(keys)
44
+ keys.each do |key|
45
+ log_effectful("removing #{key}")
46
+
47
+ unless dry_run?
48
+ @directory.files.get(key).destroy
49
+ end
50
+ end
51
+ end
52
+
53
+ def invalidate(keys, cdn, distribution)
54
+ keys.each_slice(1000) do |keys_slice|
55
+ keys_slice.each do |key|
56
+ log_effectful("invalidating #{key}")
57
+ end
58
+
59
+ unless dry_run?
60
+ cdn.post_invalidation(distribution, keys_slice)
61
+ end
62
+ end
63
+ end
64
+
65
+ def dry_run?
66
+ @is_dry_run
67
+ end
68
+
69
+ def log_effectful(s)
70
+ if @is_dry_run
71
+ puts "[dry run] #{s}"
72
+ else
73
+ puts s
74
+ end
75
+ end
76
+ end
77
+
24
78
  # @see Nanoc::Extra::Deployer#run
25
79
  def run
26
80
  require 'fog'
27
81
 
28
- # Get params, unsetting anything we don't want to pass through to fog.
29
82
  src = File.expand_path(source_path)
30
- bucket = config.delete(:bucket) || config.delete(:bucket_name)
31
- path = config.delete(:path)
32
- cdn_id = config.delete(:cdn_id)
83
+ bucket = config[:bucket] || config[:bucket_name]
84
+ path = config[:path]
85
+ cdn_id = config[:cdn_id]
33
86
 
34
- config.delete(:kind)
87
+ # FIXME: confusing error message
88
+ raise 'The path requires no trailing slash' if path && path[-1, 1] == '/'
35
89
 
36
- # Validate params
37
- error 'The path requires no trailing slash' if path && path[-1, 1] == '/'
90
+ connection = connect
91
+ directory = get_or_create_bucket(connection, bucket, path)
92
+ wrapper = FogWrapper.new(directory, dry_run?)
38
93
 
39
- # Mock if necessary
40
- if dry_run?
41
- puts 'Dry run - simulation'
42
- ::Fog.mock!
43
- end
94
+ remote_files = list_remote_files(directory)
95
+ etags = read_etags(remote_files)
44
96
 
45
- # Get connection
46
- puts 'Connecting'
47
- connection = ::Fog::Storage.new(config)
97
+ modified_keys, retained_keys = upload_all(src, path, etags, wrapper)
48
98
 
49
- # Get bucket
50
- puts 'Getting bucket'
51
- begin
52
- directory = connection.directories.get(bucket, prefix: path)
53
- rescue ::Excon::Errors::NotFound
54
- should_create_bucket = true
55
- end
56
- should_create_bucket = true if directory.nil?
99
+ removed_keys = remote_files.map(&:key) - retained_keys - modified_keys
100
+ wrapper.remove(removed_keys)
57
101
 
58
- # Create bucket if necessary
59
- if should_create_bucket
60
- puts 'Creating bucket'
61
- directory = connection.directories.create(key: bucket, prefix: path)
102
+ if cdn_id
103
+ cdn = ::Fog::CDN.new(config_for_fog)
104
+ distribution = cdn.get_distribution(cdn_id)
105
+ wrapper.invalidate(modified_keys + removed_keys, cdn, distribution)
62
106
  end
107
+ end
108
+
109
+ private
63
110
 
64
- # Get list of remote files
65
- files = directory.files
66
- truncated = files.respond_to?(:is_truncated) && files.is_truncated
67
- while truncated
68
- set = directory.files.all(marker: files.last.key)
69
- truncated = set.is_truncated
70
- files += set
111
+ def config_for_fog
112
+ config.dup.tap do |c|
113
+ c.delete(:bucket)
114
+ c.delete(:bucket_name)
115
+ c.delete(:path)
116
+ c.delete(:cdn_id)
117
+ c.delete(:kind)
71
118
  end
72
- keys_to_destroy = files.map(&:key)
73
- keys_to_invalidate = []
74
- etags = read_etags(files)
75
-
76
- # Upload all the files in the output folder to the clouds
77
- puts 'Uploading local files'
78
- FileUtils.cd(src) do
79
- files = Dir['**/*'].select { |f| File.file?(f) }
80
- files.each do |file_path|
81
- key = path ? File.join(path, file_path) : file_path
82
- upload(key, file_path, etags, keys_to_destroy, keys_to_invalidate, directory)
119
+ end
120
+
121
+ def connect
122
+ ::Fog::Storage.new(config_for_fog)
123
+ end
124
+
125
+ def get_or_create_bucket(connection, bucket, path)
126
+ directory =
127
+ begin
128
+ connection.directories.get(bucket, prefix: path)
129
+ rescue ::Excon::Errors::NotFound
130
+ nil
83
131
  end
132
+
133
+ if directory
134
+ directory
135
+ elsif dry_run?
136
+ puts '[dry run] creating bucket'
137
+ else
138
+ puts 'creating bucket'
139
+ connection.directories.create(key: bucket, prefix: path)
84
140
  end
141
+ end
142
+
143
+ def remote_key_for_local_filename(local_filename, src, path)
144
+ relative_local_filename = local_filename.sub(/\A#{src}\//, '')
85
145
 
86
- # delete extraneous remote files
87
- puts 'Removing remote files'
88
- keys_to_destroy.each do |key|
89
- directory.files.get(key).destroy
146
+ if path
147
+ File.join(path, relative_local_filename)
148
+ else
149
+ relative_local_filename
90
150
  end
151
+ end
91
152
 
92
- # invalidate CDN objects
93
- if cdn_id
94
- puts 'Invalidating CDN distribution'
95
- keys_to_invalidate.concat(keys_to_destroy)
96
- cdn = ::Fog::CDN.new(config)
97
- # fog cannot mock CDN requests
98
- unless dry_run?
99
- distribution = cdn.get_distribution(cdn_id)
100
- # usual limit per invalidation: 1000 objects
101
- keys_to_invalidate.each_slice(1000) do |paths|
102
- cdn.post_invalidation(distribution, paths)
103
- end
153
+ def list_remote_files(directory)
154
+ if directory
155
+ [].tap do |files|
156
+ directory.files.each { |file| files << file }
104
157
  end
158
+ else
159
+ []
105
160
  end
106
-
107
- puts 'Done!'
108
161
  end
109
162
 
110
- private
111
-
112
- def upload(key, file_path, etags, keys_to_destroy, keys_to_invalidate, dir)
113
- keys_to_destroy.delete(key)
163
+ def list_local_files(src)
164
+ Dir[src + '/**/*'].select { |f| File.file?(f) }
165
+ end
114
166
 
115
- return unless needs_upload?(key, file_path, etags)
167
+ def upload_all(src, path, etags, wrapper)
168
+ modified_keys = []
169
+ retained_keys = []
170
+
171
+ local_files = list_local_files(src)
172
+ local_files.each do |file_path|
173
+ key = remote_key_for_local_filename(file_path, src, path)
174
+ if needs_upload?(key, file_path, etags)
175
+ wrapper.upload(file_path, key)
176
+ modified_keys.push(key)
177
+ else
178
+ retained_keys.push(key)
179
+ end
180
+ end
116
181
 
117
- dir.files.create(
118
- key: key,
119
- body: File.open(file_path),
120
- public: true)
121
- keys_to_invalidate.push(key)
182
+ [modified_keys, retained_keys]
122
183
  end
123
184
 
124
185
  def needs_upload?(key, file_path, etags)
@@ -146,10 +207,5 @@ module Nanoc::Extra::Deployers
146
207
  Digest::MD5.file(file_path).hexdigest
147
208
  end
148
209
  end
149
-
150
- # Prints the given message on stderr and exits.
151
- def error(msg)
152
- raise RuntimeError.new(msg)
153
- end
154
210
  end
155
211
  end
@@ -32,27 +32,11 @@ module ::Nanoc::Extra
32
32
  end
33
33
 
34
34
  def filenames_per_href
35
- require 'nokogiri'
36
- filenames_per_href = {}
37
- @filenames.each do |filename|
38
- hrefs_in_file(filename).each do |href|
39
- filenames_per_href[href] ||= Set.new
40
- filenames_per_href[href] << filename
41
- end
42
- end
43
- filenames_per_href
35
+ grouped_filenames { |filename| hrefs_in_file(filename) }
44
36
  end
45
37
 
46
38
  def filenames_per_resource_uri
47
- require 'nokogiri'
48
- filenames_per_resource_uri = {}
49
- @filenames.each do |filename|
50
- resource_uris_in_file(filename).each do |resouce_uri|
51
- filenames_per_resource_uri[resouce_uri] ||= Set.new
52
- filenames_per_resource_uri[resouce_uri] << filename
53
- end
54
- end
55
- filenames_per_resource_uri
39
+ grouped_filenames { |filename| resource_uris_in_file(filename) }
56
40
  end
57
41
 
58
42
  def external_href?(href)
@@ -69,6 +53,18 @@ module ::Nanoc::Extra
69
53
 
70
54
  private
71
55
 
56
+ def grouped_filenames
57
+ require 'nokogiri'
58
+ grouped_filenames = {}
59
+ @filenames.each do |filename|
60
+ yield(filename).each do |resouce_uri|
61
+ grouped_filenames[resouce_uri] ||= Set.new
62
+ grouped_filenames[resouce_uri] << filename
63
+ end
64
+ end
65
+ grouped_filenames
66
+ end
67
+
72
68
  def uris_in_file(filename, tag_names)
73
69
  uris = Set.new
74
70
  doc = Nokogiri::HTML(::File.read(filename))
@@ -18,9 +18,9 @@ module Nanoc::Filters
18
18
  engine.render
19
19
  end
20
20
 
21
- def self.item_filename_map_for_site(site, items)
21
+ def self.item_filename_map_for_config(config, items)
22
22
  @item_filename_map ||= {}
23
- @item_filename_map[site] ||=
23
+ @item_filename_map[config] ||=
24
24
  {}.tap do |map|
25
25
  items.each do |item|
26
26
  if item.raw_filename
@@ -34,7 +34,7 @@ module Nanoc::Filters
34
34
  def imported_filename_to_item(filename)
35
35
  realpath = Pathname.new(filename).realpath.to_s
36
36
 
37
- map = self.class.item_filename_map_for_site(@site, @items)
37
+ map = self.class.item_filename_map_for_config(@config, @items)
38
38
  map[realpath]
39
39
  end
40
40
  end