protostar 0.1.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +60 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +647 -0
  5. data/THIRD_PARTY_NOTICES.md +61 -0
  6. data/exe/protoc-gen-protostar +7 -0
  7. data/exe/protostar +7 -0
  8. data/lib/protostar/builder/generator.rb +71 -0
  9. data/lib/protostar/builder/sink.rb +35 -0
  10. data/lib/protostar/builder.rb +965 -0
  11. data/lib/protostar/cli.rb +267 -0
  12. data/lib/protostar/codec/decode_generator.rb +199 -0
  13. data/lib/protostar/codec/decoder.rb +296 -0
  14. data/lib/protostar/codec/encode_generator.rb +224 -0
  15. data/lib/protostar/codec/encoder.rb +121 -0
  16. data/lib/protostar/codec/sizer.rb +91 -0
  17. data/lib/protostar/codec.rb +15 -0
  18. data/lib/protostar/compiler/ast.rb +193 -0
  19. data/lib/protostar/compiler/emitters/descriptor_set.rb +57 -0
  20. data/lib/protostar/compiler/emitters/google_ruby.rb +103 -0
  21. data/lib/protostar/compiler/emitters/grpc.rb +175 -0
  22. data/lib/protostar/compiler/emitters/protostar_rbs.rb +252 -0
  23. data/lib/protostar/compiler/emitters/rbs.rb +394 -0
  24. data/lib/protostar/compiler/emitters/ruby.rb +503 -0
  25. data/lib/protostar/compiler/emitters.rb +268 -0
  26. data/lib/protostar/compiler/include/LICENSE +32 -0
  27. data/lib/protostar/compiler/include/google/protobuf/any.proto +106 -0
  28. data/lib/protostar/compiler/include/google/protobuf/api.proto +229 -0
  29. data/lib/protostar/compiler/include/google/protobuf/compiler/plugin.proto +180 -0
  30. data/lib/protostar/compiler/include/google/protobuf/cpp_features.proto +87 -0
  31. data/lib/protostar/compiler/include/google/protobuf/descriptor.proto +1532 -0
  32. data/lib/protostar/compiler/include/google/protobuf/duration.proto +115 -0
  33. data/lib/protostar/compiler/include/google/protobuf/empty.proto +51 -0
  34. data/lib/protostar/compiler/include/google/protobuf/field_mask.proto +243 -0
  35. data/lib/protostar/compiler/include/google/protobuf/java_features.proto +132 -0
  36. data/lib/protostar/compiler/include/google/protobuf/source_context.proto +48 -0
  37. data/lib/protostar/compiler/include/google/protobuf/struct.proto +111 -0
  38. data/lib/protostar/compiler/include/google/protobuf/timestamp.proto +145 -0
  39. data/lib/protostar/compiler/include/google/protobuf/type.proto +217 -0
  40. data/lib/protostar/compiler/include/google/protobuf/wrappers.proto +157 -0
  41. data/lib/protostar/compiler/lexer.rb +659 -0
  42. data/lib/protostar/compiler/linker.rb +894 -0
  43. data/lib/protostar/compiler/option_interpreter.rb +363 -0
  44. data/lib/protostar/compiler/parser.rb +1696 -0
  45. data/lib/protostar/compiler/retention.rb +104 -0
  46. data/lib/protostar/compiler/run.rb +184 -0
  47. data/lib/protostar/compiler/source_info.rb +161 -0
  48. data/lib/protostar/compiler/symbol_table.rb +223 -0
  49. data/lib/protostar/compiler/text_format.rb +340 -0
  50. data/lib/protostar/compiler/validator.rb +944 -0
  51. data/lib/protostar/compiler.rb +304 -0
  52. data/lib/protostar/diagnostics/codes.rb +1307 -0
  53. data/lib/protostar/diagnostics/formats.rb +64 -0
  54. data/lib/protostar/diagnostics/renderer.rb +240 -0
  55. data/lib/protostar/diagnostics/source.rb +153 -0
  56. data/lib/protostar/diagnostics.rb +117 -0
  57. data/lib/protostar/dsl.rb +367 -0
  58. data/lib/protostar/enum.rb +70 -0
  59. data/lib/protostar/errors.rb +23 -0
  60. data/lib/protostar/google/protobuf/any_pb.rb +90 -0
  61. data/lib/protostar/google/protobuf/api_pb.rb +199 -0
  62. data/lib/protostar/google/protobuf/compiler/plugin_pb.rb +168 -0
  63. data/lib/protostar/google/protobuf/descriptor_pb.rb +1239 -0
  64. data/lib/protostar/google/protobuf/duration_pb.rb +99 -0
  65. data/lib/protostar/google/protobuf/empty_pb.rb +36 -0
  66. data/lib/protostar/google/protobuf/field_mask_pb.rb +228 -0
  67. data/lib/protostar/google/protobuf/source_context_pb.rb +33 -0
  68. data/lib/protostar/google/protobuf/struct_pb.rb +91 -0
  69. data/lib/protostar/google/protobuf/timestamp_pb.rb +129 -0
  70. data/lib/protostar/google/protobuf/type_pb.rb +202 -0
  71. data/lib/protostar/google/protobuf/wrappers_pb.rb +125 -0
  72. data/lib/protostar/google_protobuf_shim.rb +24 -0
  73. data/lib/protostar/json/encoder.rb +340 -0
  74. data/lib/protostar/json/errors.rb +34 -0
  75. data/lib/protostar/json/parser.rb +437 -0
  76. data/lib/protostar/json/path.rb +39 -0
  77. data/lib/protostar/json.rb +95 -0
  78. data/lib/protostar/lint/command.rb +197 -0
  79. data/lib/protostar/lint/config.rb +98 -0
  80. data/lib/protostar/lint/context.rb +126 -0
  81. data/lib/protostar/lint/directives.rb +56 -0
  82. data/lib/protostar/lint/fixer.rb +165 -0
  83. data/lib/protostar/lint/naming.rb +54 -0
  84. data/lib/protostar/lint/rule.rb +120 -0
  85. data/lib/protostar/lint/rules/comment_enum.rb +31 -0
  86. data/lib/protostar/lint/rules/comment_enum_value.rb +32 -0
  87. data/lib/protostar/lint/rules/comment_field.rb +33 -0
  88. data/lib/protostar/lint/rules/comment_message.rb +31 -0
  89. data/lib/protostar/lint/rules/comment_rpc.rb +31 -0
  90. data/lib/protostar/lint/rules/comment_service.rb +31 -0
  91. data/lib/protostar/lint/rules/edition_recommended.rb +31 -0
  92. data/lib/protostar/lint/rules/enum_first_value_zero.rb +34 -0
  93. data/lib/protostar/lint/rules/enum_pascal_case.rb +25 -0
  94. data/lib/protostar/lint/rules/enum_value_prefix.rb +38 -0
  95. data/lib/protostar/lint/rules/enum_value_upper_snake_case.rb +28 -0
  96. data/lib/protostar/lint/rules/enum_zero_value_suffix.rb +41 -0
  97. data/lib/protostar/lint/rules/field_lower_snake_case.rb +34 -0
  98. data/lib/protostar/lint/rules/field_not_required.rb +39 -0
  99. data/lib/protostar/lint/rules/field_number_reuse_risk.rb +45 -0
  100. data/lib/protostar/lint/rules/field_numbers_hot_range.rb +44 -0
  101. data/lib/protostar/lint/rules/file_lower_snake_case.rb +27 -0
  102. data/lib/protostar/lint/rules/import_no_public.rb +29 -0
  103. data/lib/protostar/lint/rules/import_no_weak.rb +28 -0
  104. data/lib/protostar/lint/rules/import_used.rb +33 -0
  105. data/lib/protostar/lint/rules/message_pascal_case.rb +27 -0
  106. data/lib/protostar/lint/rules/no_deprecated_features.rb +50 -0
  107. data/lib/protostar/lint/rules/no_groups.rb +34 -0
  108. data/lib/protostar/lint/rules/oneof_lower_snake_case.rb +27 -0
  109. data/lib/protostar/lint/rules/package_defined.rb +28 -0
  110. data/lib/protostar/lint/rules/package_directory_match.rb +33 -0
  111. data/lib/protostar/lint/rules/package_lower_snake_case.rb +28 -0
  112. data/lib/protostar/lint/rules/package_same_directory.rb +39 -0
  113. data/lib/protostar/lint/rules/package_version_suffix.rb +31 -0
  114. data/lib/protostar/lint/rules/reserved_names_with_numbers.rb +44 -0
  115. data/lib/protostar/lint/rules/rpc_pascal_case.rb +25 -0
  116. data/lib/protostar/lint/rules/rpc_request_response_unique.rb +40 -0
  117. data/lib/protostar/lint/rules/rpc_request_standard_name.rb +32 -0
  118. data/lib/protostar/lint/rules/rpc_response_standard_name.rb +31 -0
  119. data/lib/protostar/lint/rules/service_pascal_case.rb +25 -0
  120. data/lib/protostar/lint/rules/service_suffix.rb +28 -0
  121. data/lib/protostar/lint/rules/syntax_specified.rb +35 -0
  122. data/lib/protostar/lint/runner.rb +135 -0
  123. data/lib/protostar/lint.rb +122 -0
  124. data/lib/protostar/message/accessors.rb +86 -0
  125. data/lib/protostar/message/coercion.rb +171 -0
  126. data/lib/protostar/message/field_info.rb +145 -0
  127. data/lib/protostar/message/reflection.rb +149 -0
  128. data/lib/protostar/message.rb +572 -0
  129. data/lib/protostar/plugin.rb +94 -0
  130. data/lib/protostar/schema/default_value.rb +113 -0
  131. data/lib/protostar/schema/descriptor.rb +41 -0
  132. data/lib/protostar/schema/draft/copy.rb +88 -0
  133. data/lib/protostar/schema/draft/resolution.rb +177 -0
  134. data/lib/protostar/schema/draft.rb +779 -0
  135. data/lib/protostar/schema/enum.rb +106 -0
  136. data/lib/protostar/schema/extension_range.rb +37 -0
  137. data/lib/protostar/schema/features.rb +153 -0
  138. data/lib/protostar/schema/field.rb +201 -0
  139. data/lib/protostar/schema/file.rb +217 -0
  140. data/lib/protostar/schema/freezer.rb +61 -0
  141. data/lib/protostar/schema/message.rb +104 -0
  142. data/lib/protostar/schema/oneof.rb +38 -0
  143. data/lib/protostar/schema/options.rb +367 -0
  144. data/lib/protostar/schema/pool.rb +179 -0
  145. data/lib/protostar/schema/proto.rb +576 -0
  146. data/lib/protostar/schema/resolver.rb +118 -0
  147. data/lib/protostar/schema/service.rb +89 -0
  148. data/lib/protostar/schema/source_info.rb +53 -0
  149. data/lib/protostar/schema.rb +65 -0
  150. data/lib/protostar/short_names/any.rb +7 -0
  151. data/lib/protostar/short_names/duration.rb +7 -0
  152. data/lib/protostar/short_names/empty.rb +7 -0
  153. data/lib/protostar/short_names/field_mask.rb +7 -0
  154. data/lib/protostar/short_names/struct.rb +9 -0
  155. data/lib/protostar/short_names/timestamp.rb +7 -0
  156. data/lib/protostar/short_names/wrappers.rb +15 -0
  157. data/lib/protostar/text/extensions.rb +114 -0
  158. data/lib/protostar/text/parser.rb +439 -0
  159. data/lib/protostar/text/printer.rb +304 -0
  160. data/lib/protostar/text/tokenizer.rb +116 -0
  161. data/lib/protostar/text.rb +67 -0
  162. data/lib/protostar/types.rb +152 -0
  163. data/lib/protostar/unknown_field.rb +58 -0
  164. data/lib/protostar/version.rb +5 -0
  165. data/lib/protostar/view/generator.rb +298 -0
  166. data/lib/protostar/view/map.rb +194 -0
  167. data/lib/protostar/view/repeated.rb +143 -0
  168. data/lib/protostar/view/scan.rb +183 -0
  169. data/lib/protostar/view.rb +283 -0
  170. data/lib/protostar/well_known/any.rb +52 -0
  171. data/lib/protostar/well_known/duration.rb +53 -0
  172. data/lib/protostar/well_known/list_value.rb +63 -0
  173. data/lib/protostar/well_known/struct.rb +55 -0
  174. data/lib/protostar/well_known/timestamp.rb +64 -0
  175. data/lib/protostar/well_known/value.rb +60 -0
  176. data/lib/protostar/well_known.rb +38 -0
  177. data/lib/protostar/wire/io_source.rb +40 -0
  178. data/lib/protostar/wire/reader.rb +254 -0
  179. data/lib/protostar/wire/writer.rb +201 -0
  180. data/lib/protostar/wire.rb +96 -0
  181. data/lib/protostar.rb +95 -0
  182. data/shims/google-protobuf/lib/google/protobuf/any_pb.rb +17 -0
  183. data/shims/google-protobuf/lib/google/protobuf/api_pb.rb +22 -0
  184. data/shims/google-protobuf/lib/google/protobuf/compiler/plugin_pb.rb +25 -0
  185. data/shims/google-protobuf/lib/google/protobuf/descriptor_pb.rb +72 -0
  186. data/shims/google-protobuf/lib/google/protobuf/descriptor_pool.rb +164 -0
  187. data/shims/google-protobuf/lib/google/protobuf/descriptors.rb +553 -0
  188. data/shims/google-protobuf/lib/google/protobuf/duration_pb.rb +17 -0
  189. data/shims/google-protobuf/lib/google/protobuf/empty_pb.rb +17 -0
  190. data/shims/google-protobuf/lib/google/protobuf/field_mask_pb.rb +17 -0
  191. data/shims/google-protobuf/lib/google/protobuf/internal/convert.rb +180 -0
  192. data/shims/google-protobuf/lib/google/protobuf/internal/extensions.rb +154 -0
  193. data/shims/google-protobuf/lib/google/protobuf/internal/json.rb +104 -0
  194. data/shims/google-protobuf/lib/google/protobuf/internal/well_known.rb +140 -0
  195. data/shims/google-protobuf/lib/google/protobuf/map.rb +216 -0
  196. data/shims/google-protobuf/lib/google/protobuf/message.rb +544 -0
  197. data/shims/google-protobuf/lib/google/protobuf/message_exts.rb +41 -0
  198. data/shims/google-protobuf/lib/google/protobuf/plugin_pb.rb +4 -0
  199. data/shims/google-protobuf/lib/google/protobuf/repeated_field.rb +366 -0
  200. data/shims/google-protobuf/lib/google/protobuf/source_context_pb.rb +17 -0
  201. data/shims/google-protobuf/lib/google/protobuf/struct_pb.rb +20 -0
  202. data/shims/google-protobuf/lib/google/protobuf/timestamp_pb.rb +17 -0
  203. data/shims/google-protobuf/lib/google/protobuf/type_pb.rb +27 -0
  204. data/shims/google-protobuf/lib/google/protobuf/well_known_types.rb +37 -0
  205. data/shims/google-protobuf/lib/google/protobuf/wrappers_pb.rb +25 -0
  206. data/shims/google-protobuf/lib/google/protobuf.rb +107 -0
  207. data/shims/google-protobuf/lib/google/protobuf_ffi.rb +5 -0
  208. data/shims/google-protobuf/lib/google/protobuf_native.rb +5 -0
  209. data/sig/protostar/google/protobuf/any_pb.rbs +77 -0
  210. data/sig/protostar/google/protobuf/api_pb.rbs +242 -0
  211. data/sig/protostar/google/protobuf/compiler/plugin_pb.rbs +264 -0
  212. data/sig/protostar/google/protobuf/descriptor_pb.rbs +2220 -0
  213. data/sig/protostar/google/protobuf/duration_pb.rbs +82 -0
  214. data/sig/protostar/google/protobuf/empty_pb.rbs +18 -0
  215. data/sig/protostar/google/protobuf/field_mask_pb.rbs +170 -0
  216. data/sig/protostar/google/protobuf/source_context_pb.rbs +24 -0
  217. data/sig/protostar/google/protobuf/struct_pb.rbs +135 -0
  218. data/sig/protostar/google/protobuf/timestamp_pb.rbs +102 -0
  219. data/sig/protostar/google/protobuf/type_pb.rbs +304 -0
  220. data/sig/protostar/google/protobuf/wrappers_pb.rbs +169 -0
  221. data/sig/protostar/lint.rbs +216 -0
  222. data/sig/protostar.rbs +326 -0
  223. metadata +366 -0
@@ -0,0 +1,304 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Protostar
4
+ module Text
5
+ # Prints a message the way protoc's TextFormat::Printer does: fields in
6
+ # number order, two-space indents, map entries sorted by key, strings
7
+ # escaped with CEscape, floats in their shortest round-trip form, then
8
+ # extensions and unknown fields.
9
+ # @api private
10
+ class Printer
11
+ F = Message::FieldInfo
12
+
13
+ # CEscape: named escapes, octal for every other byte outside printable ASCII
14
+ ESCAPES = Array.new(256) do |b|
15
+ case b
16
+ when 0x0A then "\\n"
17
+ when 0x0D then "\\r"
18
+ when 0x09 then "\\t"
19
+ when 0x22 then "\\\""
20
+ when 0x27 then "\\'"
21
+ when 0x5C then "\\\\"
22
+ when 0x20..0x7E then b.chr
23
+ else format("\\%03o", b)
24
+ end.freeze
25
+ end.freeze
26
+ ESCAPED = /[^\x20-\x7e]|["'\\]/n
27
+ UTF8_ESCAPED = /[\x00-\x1f\x7f"'\\]/
28
+ ANY = "google.protobuf.Any"
29
+ FLOAT_MIN_NORMAL = 1.1754943508222875e-38
30
+
31
+ # @return [Boolean] whether protoc prints the field by its message type's
32
+ # name: a group whose field name is the type name in lower case
33
+ def self.group_like?(info)
34
+ return false unless info.kind == F::GROUP
35
+ type = info.descriptor.message_type
36
+ type.name.downcase == info.name && type.file.equal?(info.descriptor.file) &&
37
+ type.containing_message.equal?(info.descriptor.containing_message)
38
+ end
39
+
40
+ def initialize(single_line: false, expand_any: false, as_utf8: false, unknown_fields: true, registry: nil, pool: nil)
41
+ @single_line = single_line
42
+ @unknown_fields = unknown_fields
43
+ @expand_any = expand_any
44
+ @as_utf8 = as_utf8
45
+ @registry = registry
46
+ @extensions = Extensions.new(registry, pool)
47
+ end
48
+
49
+ # @param message [Message]
50
+ # @return [String]
51
+ def print(message)
52
+ out = +""
53
+ fields(message, out, 0)
54
+ @single_line ? out.rstrip : out
55
+ end
56
+
57
+ private
58
+
59
+ def fields(msg, out, depth)
60
+ return if @expand_any && any(msg, out, depth)
61
+ values = msg.__values
62
+ items = []
63
+ msg.class.__fields.each do |info|
64
+ v = values[info.index]
65
+ next if v.nil?
66
+ if info.map || info.repeated
67
+ next if v.empty?
68
+ elsif !info.presence && info.zero?(v)
69
+ next
70
+ end
71
+ items << [info, v]
72
+ end
73
+ unknown = msg.unknown_fields
74
+ unless unknown.empty?
75
+ extensions = @extensions.of(msg.class.descriptor)
76
+ unless extensions.empty?
77
+ unknown = extensions_as_items(unknown, extensions, items)
78
+ end
79
+ end
80
+ items.sort_by! { |info, _| info.number }
81
+ items.each { |info, v| field(info, v, out, depth) }
82
+ unknown_fields(unknown.map { |u| [u.number, u.wire_type, u] }, out, depth) if @unknown_fields
83
+ end
84
+
85
+ # Moves the unknown fields that hold extensions into items; returns the rest
86
+ def extensions_as_items(unknown, extensions, items)
87
+ by_number = unknown.group_by(&:number)
88
+ rest = unknown
89
+ by_number.each do |number, list|
90
+ field = extensions[number] or next
91
+ info = @extensions.info(field) or next
92
+ values = @extensions.read(info, list) or next
93
+ next if values.empty?
94
+ items << [info, info.repeated ? values : values.last]
95
+ rest -= list
96
+ end
97
+ rest
98
+ end
99
+
100
+ def field(info, v, out, depth)
101
+ extension = info.descriptor.extension?
102
+ name = if extension then "[#{info.descriptor.full_name}]"
103
+ elsif self.class.group_like?(info) then info.descriptor.message_type.name
104
+ else info.name
105
+ end
106
+ if info.map
107
+ v.keys.sort_by { |k| map_key_order(k) }.each do |k|
108
+ open_block(name, out, depth)
109
+ line("key: #{scalar(info.key, k)}", out, depth + 1)
110
+ value = v[k]
111
+ if info.value.message?
112
+ open_block("value", out, depth + 1)
113
+ fields(value, out, depth + 2)
114
+ close_block(out, depth + 1)
115
+ else
116
+ line("value: #{scalar(info.value, value)}", out, depth + 1)
117
+ end
118
+ close_block(out, depth)
119
+ end
120
+ else
121
+ (info.repeated ? v : [v]).each do |e|
122
+ if info.message?
123
+ open_block(name, out, depth)
124
+ fields(e, out, depth + 1)
125
+ close_block(out, depth)
126
+ else
127
+ line("#{name}: #{scalar(info, e)}", out, depth)
128
+ end
129
+ end
130
+ end
131
+ end
132
+
133
+ def map_key_order(k)
134
+ case k
135
+ when false then 0
136
+ when true then 1
137
+ else k
138
+ end
139
+ end
140
+
141
+ def line(text, out, depth)
142
+ if @single_line
143
+ out << text << " "
144
+ else
145
+ out << (" " * depth) << text << "\n"
146
+ end
147
+ end
148
+
149
+ def open_block(name, out, depth)
150
+ line("#{name} {", out, depth)
151
+ end
152
+
153
+ def close_block(out, depth)
154
+ line("}", out, depth)
155
+ end
156
+
157
+ def scalar(info, v)
158
+ case info.kind
159
+ when F::DOUBLE then double(v)
160
+ when F::FLOAT then float(v)
161
+ when F::BOOL then v ? "true" : "false"
162
+ when F::STRING then quote(v, @as_utf8)
163
+ when F::BYTES then quote(v, false)
164
+ when F::ENUM then v.to_s
165
+ else v.to_s
166
+ end
167
+ end
168
+
169
+ def quote(s, utf8)
170
+ if utf8 && s.valid_encoding? && s.encoding == Encoding::UTF_8
171
+ "\"#{s.gsub(UTF8_ESCAPED) { |c| ESCAPES[c.ord] }}\""
172
+ else
173
+ "\"#{s.b.gsub(ESCAPED) { |c| ESCAPES[c.ord] }}\""
174
+ end
175
+ end
176
+
177
+ def double(v)
178
+ return nonfinite(v) unless v.finite?
179
+ s = format("%.15g", v)
180
+ back = Float(s)
181
+ (back == v) ? s : format("%.17g", v)
182
+ end
183
+
184
+ # protoc reads the short form back with strtof, which it does not trust
185
+ # for subnormal results, so those always get nine digits
186
+ def float(v)
187
+ return nonfinite(v) unless v.finite?
188
+ s = format("%.6g", v)
189
+ back = Message::Coercion.float32(Float(s))
190
+ (back == v && (back == 0 || back.abs >= FLOAT_MIN_NORMAL)) ? s : format("%.9g", v)
191
+ end
192
+
193
+ def nonfinite(v)
194
+ if v.nan? then "nan"
195
+ elsif v > 0 then "inf"
196
+ else "-inf"
197
+ end
198
+ end
199
+
200
+ def any(msg, out, depth)
201
+ return false unless msg.class.descriptor.full_name == ANY
202
+ values = msg.__values
203
+ by_name = msg.class.__by_name
204
+ url = values[by_name.fetch("type_url").index]
205
+ bytes = values[by_name.fetch("value").index]
206
+ return false if url.nil? || url.empty? || !msg.unknown_fields.empty?
207
+ slash = url.rindex("/")
208
+ return false if slash.nil?
209
+ klass = Types.find(url[(slash + 1)..], registry: @registry || Types.registry)
210
+ return false unless klass.is_a?(Class)
211
+ inner = begin
212
+ klass.decode(bytes || "")
213
+ rescue DecodeError
214
+ return false
215
+ end
216
+ open_block("[#{url}]", out, depth)
217
+ fields(inner, out, depth + 1)
218
+ close_block(out, depth)
219
+ true
220
+ end
221
+
222
+ # protoc's PrintUnknownFields: varints unsigned, fixed widths in hex,
223
+ # length-delimited as a nested block when the bytes read as fields
224
+ def unknown_fields(list, out, depth)
225
+ list.each do |number, wire_type, value|
226
+ case wire_type
227
+ when Wire::VARINT
228
+ line("#{number}: #{unknown_value(wire_type, value)}", out, depth)
229
+ when Wire::FIXED32, Wire::FIXED64
230
+ line("#{number}: #{unknown_value(wire_type, value)}", out, depth)
231
+ when Wire::LENGTH_DELIMITED
232
+ payload = value.is_a?(UnknownField) ? value.value : value
233
+ nested = payload.empty? ? nil : UnknownSet.parse(payload)
234
+ if nested
235
+ open_block(number, out, depth)
236
+ unknown_fields(nested, out, depth + 1)
237
+ close_block(out, depth)
238
+ else
239
+ line("#{number}: #{quote(payload, false)}", out, depth)
240
+ end
241
+ when Wire::START_GROUP
242
+ nested = value.is_a?(UnknownField) ? UnknownSet.parse(value.value) : value
243
+ open_block(number, out, depth)
244
+ unknown_fields(nested || [], out, depth + 1)
245
+ close_block(out, depth)
246
+ end
247
+ end
248
+ end
249
+
250
+ def unknown_value(wire_type, value)
251
+ value = value.value if value.is_a?(UnknownField)
252
+ case wire_type
253
+ when Wire::FIXED32 then format("0x%08x", value)
254
+ when Wire::FIXED64 then format("0x%016x", value)
255
+ else value.to_s
256
+ end
257
+ end
258
+
259
+ # Reads bytes as a sequence of fields with no schema, the way protoc's
260
+ # UnknownFieldSet does, to see whether a length-delimited field holds a message
261
+ # @api private
262
+ module UnknownSet
263
+ MAX_DEPTH = 100
264
+
265
+ module_function
266
+
267
+ # @return [Array<Array(Integer, Integer, Object)>, nil] number, wire type and
268
+ # value (Integer, payload String, or the fields of a group); nil when the
269
+ # bytes are not a well-formed sequence of fields
270
+ def parse(bytes)
271
+ r = Wire::Reader.new(bytes)
272
+ list = read(r, nil, MAX_DEPTH)
273
+ r.eof? ? list : nil
274
+ rescue DecodeError
275
+ nil
276
+ end
277
+
278
+ def read(r, group, depth)
279
+ return nil if depth <= 0
280
+ list = []
281
+ until r.eof?
282
+ tag = r.read_varint
283
+ number = tag >> 3
284
+ wire_type = tag & 7
285
+ return nil if number == 0 || number > Wire::MAX_FIELD_NUMBER
286
+ case wire_type
287
+ when Wire::VARINT then list << [number, wire_type, r.read_varint]
288
+ when Wire::FIXED64 then list << [number, wire_type, r.read_fixed64]
289
+ when Wire::FIXED32 then list << [number, wire_type, r.read_fixed32]
290
+ when Wire::LENGTH_DELIMITED then list << [number, wire_type, r.read_bytes]
291
+ when Wire::START_GROUP
292
+ nested = read(r, number, depth - 1) or return nil
293
+ list << [number, wire_type, nested]
294
+ when Wire::END_GROUP
295
+ return (number == group) ? list : nil
296
+ else return nil
297
+ end
298
+ end
299
+ group ? nil : list
300
+ end
301
+ end
302
+ end
303
+ end
304
+ end
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "strscan"
4
+
5
+ module Protostar
6
+ module Text
7
+ # Splits text format into tokens the way protoc's io::Tokenizer does:
8
+ # identifiers, integers (decimal, hex, octal), floats (an `f` suffix
9
+ # allowed), quoted strings, and single-character symbols. `#` starts a
10
+ # comment that runs to the end of the line.
11
+ # @api private
12
+ class Tokenizer
13
+ # One token: `type` is :identifier, :integer, :float, :string, :symbol
14
+ # or :eof, `text` its source text, `pos` its byte offset
15
+ class Token
16
+ attr_reader :type, :text, :pos
17
+
18
+ def initialize(type, text, pos)
19
+ @type = type
20
+ @text = text
21
+ @pos = pos
22
+ freeze
23
+ end
24
+
25
+ def ==(other)
26
+ other.is_a?(Token) && other.type == @type && other.text == @text && other.pos == @pos
27
+ end
28
+ end
29
+
30
+ SKIP = /(?:\s+|#[^\n]*)+/
31
+ IDENTIFIER = /[A-Za-z_][A-Za-z0-9_]*/
32
+ HEX = /0[xX][0-9A-Fa-f]+/
33
+ FLOAT = /(?:\d+\.\d*|\.\d+)(?:[eE][-+]?\d+)?[fF]?|\d+[eE][-+]?\d+[fF]?|\d+[fF]/
34
+ INTEGER = /\d+/
35
+ STRING = /"(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*'/
36
+ NUMBER_TAIL = /[A-Za-z0-9_.]/
37
+ URL_CHARS = %r{[A-Za-z0-9_\-.~!$&()*+,;=%/]+}
38
+
39
+ def initialize(text)
40
+ @text = text
41
+ @scanner = StringScanner.new(text)
42
+ @peeked = nil
43
+ end
44
+
45
+ # @return [Token] the next token without consuming it
46
+ def peek
47
+ @peeked ||= scan
48
+ end
49
+
50
+ # @return [Token] the next token, consumed
51
+ def next
52
+ t = peek
53
+ @peeked = nil
54
+ t
55
+ end
56
+
57
+ # What protoc's tokenizer reads in URL mode between `[` and `]`: runs of
58
+ # URL characters, joined, with whitespace and comments between them
59
+ # dropped. Call it right after consuming the `[`.
60
+ # @return [Array(String, Integer)] the text and where it starts
61
+ def url_chars
62
+ raise ArgumentError, "a token was peeked past the [" if @peeked
63
+ @scanner.skip(SKIP)
64
+ pos = @scanner.pos
65
+ text = +""
66
+ while (run = @scanner.scan(URL_CHARS))
67
+ text << run
68
+ @scanner.skip(SKIP)
69
+ end
70
+ [text, pos]
71
+ end
72
+
73
+ # @return [Array(Integer, Integer)] 1-based line and column of a byte position
74
+ def line_col(pos)
75
+ before = @text.byteslice(0, pos)
76
+ line = before.count("\n") + 1
77
+ last = before.rindex("\n")
78
+ [line, before.length - (last ? last + 1 : 0) + 1]
79
+ end
80
+
81
+ # @raise [ParseError]
82
+ def fail!(message, pos)
83
+ raise ParseError.new(message, *line_col(pos))
84
+ end
85
+
86
+ private
87
+
88
+ def scan
89
+ s = @scanner
90
+ s.skip(SKIP)
91
+ pos = s.pos
92
+ return Token.new(:eof, "", pos) if s.eos?
93
+ if (t = s.scan(IDENTIFIER))
94
+ Token.new(:identifier, t, pos)
95
+ elsif (t = s.scan(HEX))
96
+ number(:integer, t, pos)
97
+ elsif (t = s.scan(FLOAT))
98
+ number(:float, t, pos)
99
+ elsif (t = s.scan(INTEGER))
100
+ number(:integer, t, pos)
101
+ elsif (t = s.scan(STRING))
102
+ Token.new(:string, t, pos)
103
+ elsif s.check(/["']/)
104
+ fail!("unterminated string", pos)
105
+ else
106
+ Token.new(:symbol, s.getch, pos)
107
+ end
108
+ end
109
+
110
+ def number(type, text, pos)
111
+ fail!("need space between number and identifier", @scanner.pos) if @scanner.check(NUMBER_TAIL)
112
+ Token.new(type, text, pos)
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Protostar
4
+ # protoc's text format: what `protoc --decode` prints and `--encode` reads.
5
+ #
6
+ # text = Protostar::Text.print(person) # also person.to_text
7
+ # Protostar::Text.parse(Person, text) == person
8
+ #
9
+ # Extensions have no API of their own yet: they travel as unknown fields.
10
+ # The printer shows an unknown field as `[full.name]` when it is an
11
+ # extension of the message known to its file (or to `pool:`), and the
12
+ # parser turns `[full.name]` into the unknown field the binary codec would
13
+ # have kept.
14
+ module Text
15
+ # Raised by everything in {Text}
16
+ class Error < Protostar::Error; end
17
+
18
+ # Raised for text that does not describe a message of the class it is
19
+ # parsed into. The message starts with `line:column:`, both 1-based.
20
+ class ParseError < Error
21
+ # @return [Integer]
22
+ attr_reader :line, :column
23
+
24
+ def initialize(message, line, column)
25
+ @line = line
26
+ @column = column
27
+ super("#{line}:#{column}: #{message}")
28
+ end
29
+ end
30
+
31
+ class << self
32
+ # @param message [Message]
33
+ # @option options [Boolean] :single_line everything on one line, fields separated by spaces
34
+ # @option options [Boolean] :expand_any print an Any whose type is known as
35
+ # `[type_url] { ... }`; off by default, as in `protoc --decode`
36
+ # (protoc's DebugString expands)
37
+ # @option options [Boolean] :as_utf8 keep valid UTF-8 in strings readable instead of
38
+ # escaping every byte above 0x7e in octal as protoc does
39
+ # @option options [Boolean] :unknown_fields false leaves unknown fields out, as
40
+ # protoc's Printer#SetHideUnknownFields does; extensions still print
41
+ # @option options [Types::Registry] :registry where classes for Any and extensions are found
42
+ # @option options [Schema::Pool, nil] :pool where extensions are found; by default
43
+ # the message's file and the files it imports
44
+ # @return [String]
45
+ def print(message, **options)
46
+ Printer.new(**options).print(message)
47
+ end
48
+
49
+ # @param klass [Class] a message class
50
+ # @param text [String]
51
+ # @option options [Boolean] :allow_unknown_fields skip fields and extensions that are not known
52
+ # @option options [Boolean] :allow_partial when false, raise if a required field is missing
53
+ # @option options [Types::Registry] :registry where classes for Any and extensions are found
54
+ # @option options [Schema::Pool, nil] :pool where extensions are found
55
+ # @return [Message]
56
+ # @raise [ParseError]
57
+ def parse(klass, text, **options)
58
+ Parser.new(text, **options).parse(klass)
59
+ end
60
+ end
61
+ end
62
+ end
63
+
64
+ require_relative "text/tokenizer"
65
+ require_relative "text/extensions"
66
+ require_relative "text/printer"
67
+ require_relative "text/parser"
@@ -0,0 +1,152 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Protostar
4
+ # Defines message classes and enum modules from descriptors, and keeps a
5
+ # registry of them by full name.
6
+ #
7
+ # types = Protostar::Types.define(schema_file)
8
+ # Person = types.fetch("acme.Person")
9
+ # Protostar::Types.find("acme.Person") # => Person
10
+ #
11
+ # The registry is a process-wide object guarded by a Mutex and is meant to be
12
+ # used from the main Ractor. The classes and modules it hands out can be used
13
+ # from any Ractor.
14
+ module Types
15
+ # A set of defined types, by full name, and the files they came from
16
+ class Registry
17
+ # @param open_enums [Boolean] treat every enum as open: numbers without a
18
+ # name are kept in the field on decode and accepted on assignment, as
19
+ # google-protobuf's Ruby gem does, instead of going to unknown fields
20
+ def initialize(open_enums: false)
21
+ @open_enums = open_enums
22
+ @mutex = Mutex.new
23
+ @types = {}
24
+ @files = {}
25
+ end
26
+
27
+ # @param full_name [String] without a leading dot
28
+ # @return [Class, Module, nil]
29
+ def find(full_name)
30
+ @mutex.synchronize { @types[full_name] }
31
+ end
32
+
33
+ # @param file_name [String]
34
+ # @return [Hash{String => Class, Module}, nil] what {#define} returned for the file
35
+ def file(file_name)
36
+ @mutex.synchronize { @files[file_name]&.last }
37
+ end
38
+
39
+ # Defines every message and enum of a file. The file's dependencies must
40
+ # have been defined first. Defining the same {Schema::File} again returns
41
+ # the types made the first time.
42
+ # @param file [Schema::File]
43
+ # @param name [Module, nil] a namespace: when given, top-level types
44
+ # become its constants and nested types constants of their parent class
45
+ # @return [Hash{String => Class, Module}] frozen, full name to class or module
46
+ # @raise [Error] for an undefined dependency or a name already defined from another file
47
+ def define(file, name: nil)
48
+ @mutex.synchronize do
49
+ if (defined = @files[file.name])
50
+ return defined.last if defined.first.equal?(file)
51
+ raise Error, "file \"#{file.name}\" is already defined from another descriptor"
52
+ end
53
+ types = Definer.new(file, @types, open_enums: @open_enums).define
54
+ types.each_key do |full_name|
55
+ raise Error, "\"#{full_name}\" is already defined by \"#{@types[full_name].descriptor.file.name}\"" if @types.key?(full_name)
56
+ end
57
+ Types.assign_constants(file, types, name) if name
58
+ types.each_value { |t| t.freeze if t.is_a?(Module) && !t.is_a?(Class) }
59
+ @types.merge!(types)
60
+ @files[file.name] = [file, types]
61
+ types
62
+ end
63
+ end
64
+ end
65
+
66
+ @registry = Registry.new
67
+
68
+ class << self
69
+ # @return [Registry] the process-wide registry
70
+ attr_reader :registry
71
+
72
+ # @see Registry#define
73
+ def define(file, name: nil, registry: @registry)
74
+ registry.define(file, name: name)
75
+ end
76
+
77
+ # @see Registry#find
78
+ def find(full_name, registry: @registry)
79
+ registry.find(full_name)
80
+ end
81
+
82
+ # @api private
83
+ def assign_constants(file, types, namespace)
84
+ place = lambda do |descriptor, owner|
85
+ const = Enum.constant_name(descriptor.name)
86
+ type = types[descriptor.full_name]
87
+ return unless const && type
88
+ owner.const_set(const, type) unless owner.const_defined?(const, false)
89
+ return unless descriptor.is_a?(Schema::Message)
90
+ descriptor.nested_messages.each { |m| place.call(m, type) }
91
+ descriptor.nested_enums.each { |e| place.call(e, type) }
92
+ end
93
+ file.messages.each { |m| place.call(m, namespace) }
94
+ file.enums.each { |e| place.call(e, namespace) }
95
+ end
96
+ end
97
+
98
+ # Makes the classes and modules of one file
99
+ # @api private
100
+ class Definer
101
+ def initialize(file, known, open_enums: false)
102
+ @file = file
103
+ @known = known
104
+ @open_enums = open_enums
105
+ @types = {}
106
+ @reserved = Message::Accessors.reserved
107
+ end
108
+
109
+ def define
110
+ messages = @file.all_messages.reject(&:map_entry?)
111
+ @file.all_enums.each { |e| @types[e.full_name] = Enum.build(e) }
112
+ messages.each { |m| @types[m.full_name] = Class.new(Message) }
113
+ messages.each { |m| setup(@types[m.full_name], m) }
114
+ @types.freeze
115
+ end
116
+
117
+ private
118
+
119
+ # Names end up inside class_eval'd source, so anything a .proto could not spell is refused
120
+ IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/
121
+
122
+ def setup(klass, descriptor)
123
+ (descriptor.fields.map(&:name) + descriptor.real_oneofs.map(&:name)).each do |n|
124
+ raise Protostar::Error, "#{descriptor.full_name} has a name that is not an identifier: #{n.inspect}" unless IDENTIFIER.match?(n)
125
+ end
126
+ oneof_slots = descriptor.real_oneofs.to_h do |o|
127
+ [o, o.fields.map { |f| descriptor.fields.index(f) }]
128
+ end
129
+ classes = method(:resolve)
130
+ fields = descriptor.fields.each_with_index.map do |f, i|
131
+ slots = oneof_slots[f.real_oneof] if f.real_oneof
132
+ Message::FieldInfo.new(f, i, classes, siblings: slots && (slots - [i]), open_enums: @open_enums)
133
+ end
134
+ oneofs = oneof_slots.map { |o, slots| [o.name.to_sym, slots] }
135
+ klass.__define(descriptor: descriptor, fields: fields, oneofs: oneofs)
136
+ Message::Accessors.install(klass, fields, oneofs, @reserved)
137
+ Codec::DecodeGenerator.install(klass, fields)
138
+ Codec::EncodeGenerator.install(klass, fields)
139
+ View::Generator.install(klass, fields, oneofs)
140
+ return unless descriptor.full_name.start_with?("google.protobuf.")
141
+ well_known = WellKnown::MIXINS[descriptor.full_name]
142
+ klass.include(well_known) if well_known
143
+ end
144
+
145
+ def resolve(descriptor)
146
+ @types[descriptor.full_name] || @known[descriptor.full_name] ||
147
+ raise(Error, "#{descriptor.full_name} comes from \"#{descriptor.file.name}\", which must be " \
148
+ "defined before \"#{@file.name}\"")
149
+ end
150
+ end
151
+ end
152
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Protostar
4
+ # A field the decoder did not recognise, kept so that it is re-emitted on
5
+ # encode. {#bytes} are the raw bytes that followed the tag on the wire,
6
+ # verbatim: a varint as it was encoded, 4 or 8 fixed bytes, a length prefix
7
+ # and its payload, or a group's contents through its end tag.
8
+ class UnknownField
9
+ # @return [Integer]
10
+ attr_reader :number
11
+ # @return [Integer] one of the {Wire} wire type constants
12
+ attr_reader :wire_type
13
+ # @return [String] frozen, binary
14
+ attr_reader :bytes
15
+
16
+ # @param number [Integer]
17
+ # @param wire_type [Integer]
18
+ # @param bytes [String] the bytes after the tag; copied unless frozen and binary
19
+ def initialize(number, wire_type, bytes)
20
+ @number = number
21
+ @wire_type = wire_type
22
+ @bytes = (bytes.frozen? && bytes.encoding == Encoding::BINARY) ? bytes : bytes.b.freeze
23
+ freeze
24
+ end
25
+
26
+ # @return [Integer, String] the decoded value: an Integer for varint and
27
+ # fixed wire types, the payload without its length prefix for
28
+ # length-delimited, the contents without the end tag for a group
29
+ def value
30
+ r = Wire::Reader.new(@bytes)
31
+ case @wire_type
32
+ when Wire::VARINT then r.read_varint
33
+ when Wire::FIXED32 then r.read_fixed32
34
+ when Wire::FIXED64 then r.read_fixed64
35
+ when Wire::LENGTH_DELIMITED then r.read_bytes
36
+ else @bytes.byteslice(0, @bytes.bytesize - Wire.varint_size(Wire.tag(@number, Wire::END_GROUP)))
37
+ end
38
+ end
39
+
40
+ # @return [String] the field as it goes on the wire, tag included
41
+ def to_wire
42
+ Wire.tag_bytes(@number, @wire_type) + @bytes
43
+ end
44
+
45
+ def ==(other)
46
+ other.is_a?(UnknownField) && other.number == @number && other.wire_type == @wire_type && other.bytes == @bytes
47
+ end
48
+ alias_method :eql?, :==
49
+
50
+ def hash
51
+ [UnknownField, @number, @wire_type, @bytes].hash
52
+ end
53
+
54
+ def inspect
55
+ "#<#{self.class.name} #{@number}:#{Wire::NAMES.fetch(@wire_type)} #{@bytes.inspect}>"
56
+ end
57
+ end
58
+ end