decode 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59ccb94aeeb9afb9e56f473a90991bfb2955ce6bdc2a681b1c8ecf7ee72974f9
-  data.tar.gz: d092f82acb7134df72dfbf2042617f656b94749c8f5b89ec7830e002ff781e8e
+  metadata.gz: d37f874bcef0dadaa82d665ae4f27ed182a0d45c55403a280c443dede81d1b97
+  data.tar.gz: a3394961b5285524ab07be9931aeb436cd78aa94286d6c65c81993389b162a28
 SHA512:
-  metadata.gz: 4df6fd92d34b379631cf7a0358e3d41c7470824daadd0ce9274def8b40cd5579ddb9d81eaed6cfdd8bf44a09b735525332a35b75fb0cb95068063d5e67431c52
-  data.tar.gz: fc5a74af220d01473013e7943d778017fd7f141086103fb350713e935893c2616495d1ce61a5b8421a940dfe44ca3b8eb3658dab0190ef3f4232fa2d9765892f
+  metadata.gz: 7df83797d9844a6eb0e4f52f275527327ce7d81564865edcc70d5bce99a2be89b002fbe42cd1440bf666b4197c6dbb85775b4b4097675e525c1100aaf242571a
+  data.tar.gz: b2303685a56fa40ae3195a21a7a2cf613f5301f6c4520e4e6d8853e218b44363d33d704cd487710af40728f1926832b7bfc8bbd2a90db3555f6ed22c74ec7c45

data/docs/config.ru

@@ -0,0 +1,6 @@
+
+require 'utopia/setup'
+UTOPIA ||= Utopia.setup
+
+require 'utopia/project'
+Utopia::Project.call(self)

data/docs/gems.locked

@@ -0,0 +1,137 @@
+PATH
+  remote: ../../../socketry/utopia-project
+  specs:
+    utopia-project (0.1.0)
+      decode
+      falcon
+      kramdown
+      kramdown-parser-gfm
+      rackula
+      thread-local
+      utopia (~> 2.14)
+      utopia-gallery
+
+PATH
+  remote: ..
+  specs:
+    decode (0.6.0)
+      parser
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    async (1.25.2)
+      console (~> 1.0)
+      nio4r (~> 2.3)
+      timers (~> 4.1)
+    async-container (0.16.4)
+      async (~> 1.0)
+      async-io (~> 1.26)
+      process-group
+    async-http (0.52.0)
+      async (~> 1.25)
+      async-io (~> 1.28)
+      async-pool (~> 0.2)
+      protocol-http (~> 0.18.0)
+      protocol-http1 (~> 0.12.0)
+      protocol-http2 (~> 0.14.0)
+    async-http-cache (0.2.0)
+      async-http (~> 0.51)
+    async-io (1.29.0)
+      async (~> 1.14)
+    async-pool (0.3.0)
+      async (~> 1.25)
+    build-environment (1.13.0)
+    concurrent-ruby (1.1.6)
+    console (1.8.2)
+    falcon (0.36.4)
+      async (~> 1.13)
+      async-container (~> 0.16.0)
+      async-http (~> 0.52.0)
+      async-http-cache (~> 0.2.0)
+      async-io (~> 1.22)
+      build-environment (~> 1.13)
+      localhost (~> 1.1)
+      process-metrics (~> 0.2.0)
+      rack (>= 1.0)
+      samovar (~> 2.1)
+    ffi (1.12.2)
+    http-accept (2.1.1)
+    kramdown (2.2.1)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    localhost (1.1.6)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    mapping (1.1.1)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0425)
+    mini_mime (1.0.2)
+    msgpack (1.3.3)
+    nio4r (2.5.2)
+    parser (2.7.1.2)
+      ast (~> 2.4.0)
+    process-group (1.2.1)
+      process-terminal (~> 0.2.0)
+    process-metrics (0.2.1)
+      console (~> 1.8)
+      samovar (~> 2.1)
+    process-terminal (0.2.0)
+      ffi
+    protocol-hpack (1.4.2)
+    protocol-http (0.18.0)
+    protocol-http1 (0.12.0)
+      protocol-http (~> 0.18)
+    protocol-http2 (0.14.0)
+      protocol-hpack (~> 1.4)
+      protocol-http (~> 0.18)
+    rack (2.2.2)
+    rackula (1.1.0)
+      falcon (~> 0.34)
+      samovar (~> 2.1)
+      variant
+    rake (13.0.1)
+    rake-compiler (1.1.0)
+      rake
+    rexml (3.2.4)
+    samovar (2.1.4)
+      console (~> 1.0)
+      mapping (~> 1.0)
+    thread-local (1.0.0)
+    timers (4.3.0)
+    trenni (3.9.0)
+      rake-compiler
+    utopia (2.15.1)
+      concurrent-ruby (~> 1.0)
+      console (~> 1.0)
+      http-accept (~> 2.1)
+      mail (~> 2.6)
+      mime-types (~> 3.0)
+      msgpack
+      rack (~> 2.2)
+      samovar (~> 2.1)
+      trenni (~> 3.0)
+      variant (~> 0.1)
+    utopia-gallery (2.5.0)
+      trenni (~> 3.9)
+      utopia (~> 2.0)
+      vips-thumbnail (~> 1.6)
+    variant (0.1.1)
+      thread-local
+    vips (8.9.1)
+      ffi (~> 1.9)
+    vips-thumbnail (1.6.0)
+      vips (~> 8.9)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  decode!
+  utopia-project!
+
+BUNDLED WITH
+   2.1.4

data/docs/gems.rb

@@ -0,0 +1,7 @@
+
+source "https://rubygems.org"
+
+group :preload do
+	gem "utopia-project", path: "../../../socketry/utopia-project"
+	gem "decode", path: "../"
+end

data/examples/index/extract.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+# This example demonstrates how to extract symbols using the index. An instance of {Decode::Index} is used for loading symbols from source code files. These symbols are available as a flat list and as a trie structure. You can look up specific symbols using a reference using {Decode::Index:lookup}.
+
+require_relative '../../lib/decode/index'
+
+# Firstly, construct the index:
+index = Decode::Index.new
+
+# Then, update the index by loading paths from the file system:
+paths = Dir.glob(File.expand_path("../../lib/**/*.rb", __dir__))
+index.update(paths)
+
+# Finally, you can print out the loaded symbols:
+index.symbols.each do |name, symbol|
+	puts symbol.long_form
+end
+
+# Lookup a specific symbol:
+absolute_reference = Decode::Language::Ruby.reference("Decode::Index:lookup")
+lookup_symbol = index.lookup(absolute_reference).first
+puts lookup_symbol.long_form
+
+# Lookup a method relative to that symbol:
+relative_reference = Decode::Language::Ruby.reference("trie")
+trie_attribute = index.lookup(relative_reference, relative_to: lookup_symbol).first
+puts trie_attribute.long_form

data/lib/decode/index.rb

@@ -41,10 +41,10 @@ module Decode
 				source = Source.new(path)
 				@sources[path] = Source.new(path)
 				
-				source.parse do |definition|
-					@symbols[definition.qualified_name] = definition
+				source.symbols do |symbol|
+					@symbols[symbol.qualified_name] = symbol
 					
-					@trie.insert(definition.lexical_path, definition)
+					@trie.insert(symbol.lexical_path, symbol)
 				end
 			end
 		end

data/lib/decode/language/ruby/parser.rb

@@ -27,6 +27,8 @@ require_relative 'function'
 require_relative 'method'
 require_relative 'module'
 
+require_relative 'segment'
+
 module Decode
 	module Language
 		module Ruby
@@ -35,14 +37,14 @@ module Decode
 					@parser = parser
 				end
 				
-				def parse(input, &block)
+				def symbols_for(input, &block)
 					buffer = ::Parser::Source::Buffer.new('(input)')
 					buffer.source = input.read
 					
 					top, comments = @parser.parse_with_comments(buffer)
 					
 					if top
-						walk(top, comments, &block)
+						walk_symbols(top, comments, &block)
 					end
 				end
 				
@@ -72,11 +74,11 @@ module Decode
 				end
 				
 				# Walk over the syntax tree and extract relevant definitions with their associated comments.
-				def walk(node, comments, parent = nil, &block)
+				def walk_symbols(node, comments, parent = nil, &block)
 					case node.type
 					when :begin
 						node.children.each do |child|
-							walk(child, comments, parent, &block)
+							walk_symbols(child, comments, parent, &block)
 						end
 					when :module
 						definition = Module.new(
@@ -88,7 +90,7 @@ module Decode
 						yield definition
 						
 						if children = node.children[1]
-							walk(children, comments, definition, &block)
+							walk_symbols(children, comments, definition, &block)
 						end
 					when :class
 						definition = Class.new(
@@ -100,7 +102,7 @@ module Decode
 						yield definition
 						
 						if children = node.children[2]
-							walk(children, comments, definition, &block)
+							walk_symbols(children, comments, definition, &block)
 						end
 					when :sclass
 						definition = Singleton.new(
@@ -112,7 +114,7 @@ module Decode
 						yield definition
 						
 						if children = node.children[1]
-							walk(children, comments, definition, &block)
+							walk_symbols(children, comments, definition, &block)
 						end
 					when :def
 						definition = Method.new(
@@ -159,6 +161,51 @@ module Decode
 						return node.children[0]
 					end
 				end
+				
+				def segments_for(input, &block)
+					buffer = ::Parser::Source::Buffer.new('(input)')
+					buffer.source = input.read
+					
+					top, comments = @parser.parse_with_comments(buffer)
+					
+					# We delete any leading comments:
+					line = 0
+					
+					while comment = comments.first
+						if comment.location.line == line
+							comments.pop
+							line += 1
+						else
+							break
+						end
+					end
+					
+					# Now we iterate over the syntax tree and generate segments:
+					walk_segments(top, comments, &block)
+				end
+				
+				def walk_segments(node, comments, &block)
+					case node.type
+					when :begin
+						segment = nil
+						
+						node.children.each do |child|
+							if segment.nil?
+								segment = Segment.new(
+									extract_comments_for(child, comments),
+									child
+								)
+							elsif next_comments = extract_comments_for(child, comments)
+								yield segment if segment
+								segment = Segment.new(next_comments, child)
+							else
+								segment.expand(child)
+							end
+						end
+						
+						yield segment if segment
+					end
+				end
 			end
 		end
 	end

data/lib/decode/language/ruby/segment.rb

@@ -0,0 +1,46 @@
+# Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require_relative '../../segment'
+
+module Decode
+	module Language
+		module Ruby
+			class Segment < Decode::Segment
+				def initialize(comments, node, **options)
+					super(comments, **options)
+					
+					@node = node
+					@expression = node.location.expression
+				end
+				
+				attr :node
+				
+				def expand(node)
+					@expression = @expression.join(node.location.expression)
+				end
+				
+				def code
+					@expression.source
+				end
+			end
+		end
+	end
+end

data/lib/decode/language/ruby.rb

@@ -24,8 +24,8 @@ require_relative 'ruby/parser'
 module Decode
 	module Language
 		module Ruby
-			def self.parse(input, &block)
-				Parser.new.parse(input, &block)
+			def self.name
+				"ruby"
 			end
 			
 			# The symbol which is used to separate the specified definition from the parent scope.
@@ -56,6 +56,18 @@ module Decode
 			def self.reference(value)
 				Reference.new(value)
 			end
+			
+			# Parse the input yielding symbols.
+			# @yield [Definition]
+			def self.symbols_for(input, &block)
+				Parser.new.symbols_for(input, &block)
+			end
+			
+			# Parse the input yielding interleaved comments and code segments.
+			# @yield [Segment]
+			def self.segments_for(input, &block)
+				Parser.new.segments_for(input, &block)
+			end
 		end
 	end
 end

data/lib/decode/segment.rb

@@ -0,0 +1,44 @@
+# Copyright, 2020, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require_relative 'documentation'
+
+module Decode
+	class Segment
+		def initialize(comments)
+			@comments = comments
+		end
+		
+		attr :comments
+		
+		# An interface for accsssing the documentation of the definition.
+		# @return [Documentation | nil] A `Documentation` if this definition has comments.
+		def documentation
+			if @comments&.any?
+				@documentation ||= Documentation.new(@comments)
+			end
+		end
+		
+		# The source code trailing the comments.
+		# @return [String | nil]
+		def code
+		end
+	end
+end

data/lib/decode/source.rb

@@ -27,16 +27,24 @@ module Decode
 			@language = language || Language.detect(path)
 		end
 		
-		def parse(&block)
-			return to_enum(:parse) unless block_given?
+		def open(&block)
+			File.open(@path, &block)
+		end
+		
+		def symbols(&block)
+			return to_enum(:symbols) unless block_given?
 			
 			self.open do |file|
-				@language.parse(file, &block)
+				@language.symbols_for(file, &block)
 			end
 		end
 		
-		def open(&block)
-			File.open(@path, &block)
+		def segments(&block)
+			return to_enum(:segments) unless block_given?
+			
+			self.open do |file|
+				@language.segments_for(file, &block)
+			end
 		end
 	end
 end

data/lib/decode/symbol.rb

@@ -23,6 +23,7 @@ require_relative 'documentation'
 module Decode
 	Key = Struct.new(:kind, :name)
 	
+	# A named element which represents some significant element of some kind in a computer program.
 	class Symbol
 		def initialize(kind, name, parent: nil, language: parent.language)
 			@kind = kind
@@ -34,6 +35,8 @@ module Decode
 			@qualified_name = nil
 		end
 		
+		# A symbol-specific key which represents the lexical scope.
+		# @return [Key]
 		def key
 			Key.new(@kind, @name)
 		end
@@ -42,15 +45,25 @@ module Decode
 			"\#<#{self.class} #{@kind} #{qualified_name}>"
 		end
 		
+		# The kind of symbol, e.g. `:module`.
 		attr :kind
+		
+		# The symbol name, e.g. `:Decode`.
 		attr :name
+		
+		# The parent symbol, defining lexical scope.
 		attr :parent
+		
+		# The language the symbol is defined within.
 		attr :language
 		
+		# The qualified name is an absolute name which includes any and all namespacing.
 		def qualified_name
 			@qualified_name ||= @language.join(self.path).freeze
 		end
 		
+		# The lexical scope as defined by the {key}.
+		# @return [Array]
 		def path
 			if @path
 				@path
@@ -61,6 +74,7 @@ module Decode
 			end
 		end
 		
+		# The lexical path, only taking into account the symbol names.
 		def lexical_path
 			self.path.map(&:name)
 		end

data/lib/decode/trie.rb

@@ -42,13 +42,11 @@ module Decode
 			end
 			
 			def traverse(path = [], &block)
-				catch(:skip) do
-					yield(path, self)
-					
+				yield(path, self, ->{
 					@children.each do |name, node|
 						node.traverse([*path, name], &block)
 					end
-				end
+				})
 			end
 		end
 		
@@ -71,7 +69,7 @@ module Decode
 		end
 		
 		def lookup(path)
-			@root.lookup(path).values
+			@root.lookup(path)
 		end
 		
 		# Given a base path, enumerate all paths under that.
@@ -82,6 +80,12 @@ module Decode
 			end
 		end
 		
+		def traverse(path = [], &block)
+			if node = @root.lookup(path)
+				node.traverse(&block)
+			end
+		end
+		
 		def match(path, &block)
 			@root.lookup(path)
 		end

data/lib/decode/version.rb

@@ -19,5 +19,5 @@
 # THE SOFTWARE.
 
 module Decode
-	VERSION = "0.6.0"
+	VERSION = "0.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: decode
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Samuel Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-02 00:00:00.000000000 Z
+date: 2020-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parser
@@ -120,6 +120,10 @@ files:
 - ".rspec"
 - README.md
 - decode.gemspec
+- docs/config.ru
+- docs/gems.locked
+- docs/gems.rb
+- examples/index/extract.rb
 - gems.rb
 - lib/decode.rb
 - lib/decode/definition.rb
@@ -136,6 +140,8 @@ files:
 - lib/decode/language/ruby/module.rb
 - lib/decode/language/ruby/parser.rb
 - lib/decode/language/ruby/reference.rb
+- lib/decode/language/ruby/segment.rb
+- lib/decode/segment.rb
 - lib/decode/source.rb
 - lib/decode/symbol.rb
 - lib/decode/trie.rb