protoc 2.6.1.1 → 2.6.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/protoc/Makefile.in +10 -13
- data/ext/protoc/extconf.rb +0 -1
- data/ext/protoc/protobuf/CHANGES.txt +593 -0
- data/ext/protoc/protobuf/CONTRIBUTORS.txt +93 -0
- data/ext/protoc/protobuf/INSTALL.txt +237 -0
- data/ext/protoc/protobuf/LICENSE +33 -0
- data/ext/protoc/protobuf/Makefile.am +267 -0
- data/ext/protoc/protobuf/README.md +167 -0
- data/ext/protoc/protobuf/autogen.sh +41 -0
- data/ext/protoc/protobuf/benchmarks/ProtoBench.java +203 -0
- data/ext/protoc/protobuf/benchmarks/google_message1.dat +0 -0
- data/ext/protoc/protobuf/benchmarks/google_message2.dat +0 -0
- data/ext/protoc/protobuf/benchmarks/google_size.proto +136 -0
- data/ext/protoc/protobuf/benchmarks/google_speed.proto +136 -0
- data/ext/protoc/protobuf/benchmarks/readme.txt +50 -0
- data/ext/protoc/protobuf/configure.ac +159 -0
- data/ext/protoc/protobuf/editors/README.txt +5 -0
- data/ext/protoc/protobuf/editors/proto.vim +105 -0
- data/ext/protoc/protobuf/editors/protobuf-mode.el +220 -0
- data/ext/protoc/protobuf/examples/AddPerson.java +95 -0
- data/ext/protoc/protobuf/examples/ListPeople.java +50 -0
- data/ext/protoc/protobuf/examples/Makefile +58 -0
- data/ext/protoc/protobuf/examples/README.txt +29 -0
- data/ext/protoc/protobuf/examples/add_person.cc +95 -0
- data/ext/protoc/protobuf/examples/add_person.py +58 -0
- data/ext/protoc/protobuf/examples/addressbook.proto +30 -0
- data/ext/protoc/protobuf/examples/list_people.cc +68 -0
- data/ext/protoc/protobuf/examples/list_people.py +38 -0
- data/ext/protoc/protobuf/generate_descriptor_proto.sh +33 -0
- data/ext/protoc/protobuf/java/README.txt +96 -0
- data/ext/protoc/protobuf/java/pom.xml +217 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/AbstractMessage.java +466 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/AbstractMessageLite.java +355 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/AbstractParser.java +253 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/BlockingRpcChannel.java +51 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/BlockingService.java +64 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/BoundedByteString.java +163 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/ByteString.java +1022 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/CodedInputStream.java +1311 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/CodedOutputStream.java +1297 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/Descriptors.java +2238 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/DynamicMessage.java +622 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/Extension.java +96 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/ExtensionRegistry.java +392 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/ExtensionRegistryLite.java +185 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/FieldSet.java +907 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/GeneratedMessage.java +2213 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java +949 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/Internal.java +391 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java +122 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/LazyField.java +154 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/LazyFieldLite.java +176 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/LazyStringArrayList.java +367 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/LazyStringList.java +163 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/LiteralByteString.java +362 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/Message.java +244 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/MessageLite.java +320 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/MessageLiteOrBuilder.java +60 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/MessageOrBuilder.java +143 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/MessageReflection.java +931 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/Parser.java +261 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/ProtocolMessageEnum.java +58 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/ProtocolStringList.java +48 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/RepeatedFieldBuilder.java +696 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/RopeByteString.java +957 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/RpcCallback.java +47 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/RpcChannel.java +71 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/RpcController.java +118 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/RpcUtil.java +134 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/Service.java +117 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/ServiceException.java +52 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/SingleFieldBuilder.java +241 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/SmallSortedMap.java +618 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/TextFormat.java +1984 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/UninitializedMessageException.java +99 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/UnknownFieldSet.java +995 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/UnmodifiableLazyStringList.java +205 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/Utf8.java +349 -0
- data/ext/protoc/protobuf/java/src/main/java/com/google/protobuf/WireFormat.java +163 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/AbstractMessageTest.java +527 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java +68 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/ByteStringTest.java +759 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/CheckUtf8Test.java +141 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/CodedInputStreamTest.java +769 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java +401 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/DeprecatedFieldTest.java +80 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/DescriptorsTest.java +735 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/DynamicMessageTest.java +326 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/ForceFieldBuildersPreRun.java +48 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java +1515 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/IsValidUtf8Test.java +180 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java +421 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/LazyFieldLiteTest.java +134 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/LazyFieldTest.java +121 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/LazyMessageLiteTest.java +319 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/LazyStringArrayListTest.java +174 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/LazyStringEndToEndTest.java +143 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/LiteEqualsAndHashTest.java +85 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/LiteTest.java +148 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java +396 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/MessageTest.java +353 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/NestedBuildersTest.java +185 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/ParserTest.java +381 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/RepeatedFieldBuilderTest.java +190 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java +97 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/RopeByteStringTest.java +115 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/ServiceTest.java +320 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/SingleFieldBuilderTest.java +155 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/SmallSortedMapTest.java +420 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/TestBadIdentifiers.java +96 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/TestUtil.java +4124 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/TextFormatTest.java +994 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/UnknownFieldSetTest.java +653 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/UnmodifiableLazyStringListTest.java +227 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/WireFormatTest.java +606 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/lazy_fields_lite.proto +61 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/lite_equals_and_hash.proto +55 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/multiple_files_test.proto +77 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/nested_builders_test.proto +53 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/nested_extension.proto +46 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/nested_extension_lite.proto +48 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/non_nested_extension.proto +49 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/non_nested_extension_lite.proto +50 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/outer_class_name_test.proto +38 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/outer_class_name_test2.proto +42 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/outer_class_name_test3.proto +43 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto +157 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/test_check_utf8.proto +50 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/test_check_utf8_size.proto +51 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/test_custom_options.proto +43 -0
- data/ext/protoc/protobuf/java/src/test/java/com/google/protobuf/test_extra_interfaces.proto +60 -0
- data/ext/protoc/protobuf/m4/ac_system_extensions.m4 +37 -0
- data/ext/protoc/protobuf/m4/acx_check_suncc.m4 +75 -0
- data/ext/protoc/protobuf/m4/acx_pthread.m4 +397 -0
- data/ext/protoc/protobuf/m4/stl_hash.m4 +72 -0
- data/ext/protoc/protobuf/more_tests/Makefile +41 -0
- data/ext/protoc/protobuf/post_process_dist.sh +60 -0
- data/ext/protoc/protobuf/protobuf-lite.pc.in +13 -0
- data/ext/protoc/protobuf/protobuf.pc.in +14 -0
- data/ext/protoc/protobuf/python/README.txt +105 -0
- data/ext/protoc/protobuf/python/ez_setup.py +284 -0
- data/ext/protoc/protobuf/python/google/__init__.py +1 -0
- data/ext/protoc/protobuf/python/google/protobuf/__init__.py +0 -0
- data/ext/protoc/protobuf/python/google/protobuf/descriptor.py +849 -0
- data/ext/protoc/protobuf/python/google/protobuf/descriptor_database.py +137 -0
- data/ext/protoc/protobuf/python/google/protobuf/descriptor_pool.py +643 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/__init__.py +0 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/api_implementation.cc +139 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/api_implementation.py +89 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/api_implementation_default_test.py +63 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/containers.py +269 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/cpp_message.py +663 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/decoder.py +831 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/descriptor_database_test.py +63 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/descriptor_pool_test.py +564 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/descriptor_pool_test1.proto +94 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/descriptor_pool_test2.proto +70 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/descriptor_python_test.py +54 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/descriptor_test.py +669 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/encoder.py +788 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/enum_type_wrapper.py +89 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/factory_test1.proto +57 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/factory_test2.proto +92 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/generator_test.py +343 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/message_factory_python_test.py +54 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/message_factory_test.py +131 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/message_listener.py +78 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/message_python_test.py +54 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/message_test.py +681 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/missing_enum_values.proto +50 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/more_extensions.proto +58 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/more_extensions_dynamic.proto +49 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/more_messages.proto +51 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/python_message.py +1251 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/reflection_test.py +2934 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/service_reflection_test.py +136 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/symbol_database_test.py +120 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/test_bad_identifiers.proto +52 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/test_util.py +662 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/text_encoding_test.py +68 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/text_format_test.py +743 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/type_checkers.py +328 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/unknown_fields_test.py +231 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/wire_format.py +268 -0
- data/ext/protoc/protobuf/python/google/protobuf/internal/wire_format_test.py +253 -0
- data/ext/protoc/protobuf/python/google/protobuf/message.py +284 -0
- data/ext/protoc/protobuf/python/google/protobuf/message_factory.py +155 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/README +6 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/__init__.py +0 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/cpp_message.py +61 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/descriptor.cc +357 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/descriptor.h +96 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/descriptor_cpp2_test.py +58 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/extension_dict.cc +338 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/extension_dict.h +123 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/message.cc +2561 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/message.h +305 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/message_factory_cpp2_test.py +56 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/proto2_api_test.proto +38 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/python.proto +66 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/python_protobuf.h +57 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/reflection_cpp2_generated_test.py +94 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/repeated_composite_container.cc +763 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/repeated_composite_container.h +172 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/repeated_scalar_container.cc +825 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/repeated_scalar_container.h +112 -0
- data/ext/protoc/protobuf/python/google/protobuf/pyext/scoped_pyobject_ptr.h +95 -0
- data/ext/protoc/protobuf/python/google/protobuf/reflection.py +205 -0
- data/ext/protoc/protobuf/python/google/protobuf/service.py +226 -0
- data/ext/protoc/protobuf/python/google/protobuf/service_reflection.py +284 -0
- data/ext/protoc/protobuf/python/google/protobuf/symbol_database.py +185 -0
- data/ext/protoc/protobuf/python/google/protobuf/text_encoding.py +110 -0
- data/ext/protoc/protobuf/python/google/protobuf/text_format.py +873 -0
- data/ext/protoc/protobuf/python/mox.py +1401 -0
- data/ext/protoc/protobuf/python/setup.py +201 -0
- data/ext/protoc/protobuf/python/stubout.py +140 -0
- data/ext/protoc/protobuf/src/Makefile.am +418 -0
- data/ext/protoc/protobuf/src/google/protobuf/SEBS +240 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/code_generator.cc +84 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/code_generator.h +145 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/command_line_interface.cc +1603 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/command_line_interface.h +378 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/command_line_interface_unittest.cc +1654 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_bootstrap_unittest.cc +158 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_enum.cc +288 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_enum.h +103 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_enum_field.cc +431 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_enum_field.h +122 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_extension.cc +210 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_extension.h +86 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_field.cc +166 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_field.h +185 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_file.cc +665 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_file.h +99 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_generator.cc +125 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_generator.h +72 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_helpers.cc +494 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_helpers.h +206 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_message.cc +2645 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_message.h +175 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_message_field.cc +375 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_message_field.h +121 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_options.h +58 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_plugin_unittest.cc +123 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc +451 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_primitive_field.h +123 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_service.cc +334 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_service.h +118 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_string_field.cc +642 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_string_field.h +127 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto +132 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_unittest.cc +2074 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/cpp/cpp_unittest.h +51 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/importer.cc +480 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/importer.h +317 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/importer_unittest.cc +617 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_context.cc +195 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_context.h +95 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_doc_comment.cc +233 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_doc_comment.h +69 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_doc_comment_unittest.cc +67 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_enum.cc +333 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_enum.h +99 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_enum_field.cc +778 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_enum_field.h +158 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_extension.cc +207 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_extension.h +109 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_field.cc +213 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_field.h +162 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_file.cc +534 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_file.h +115 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_generator.cc +158 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_generator.h +72 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_generator_factory.cc +77 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_generator_factory.h +101 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_helpers.cc +737 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_helpers.h +322 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_lazy_message_field.cc +826 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_lazy_message_field.h +121 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_message.cc +1666 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_message.h +140 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_message_field.cc +1343 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_message_field.h +173 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_name_resolver.cc +266 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_name_resolver.h +124 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_plugin_unittest.cc +124 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_primitive_field.cc +877 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_primitive_field.h +160 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_service.cc +473 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_service.h +135 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_shared_code_generator.cc +201 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_shared_code_generator.h +90 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_string_field.cc +1056 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/java/java_string_field.h +160 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/main.cc +61 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/mock_code_generator.cc +240 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/mock_code_generator.h +117 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/package_info.h +64 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/parser.cc +1750 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/parser.h +522 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/parser_unittest.cc +2612 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/plugin.cc +163 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/plugin.h +72 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/plugin.pb.cc +1148 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/plugin.pb.h +897 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/plugin.proto +147 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/python/python_generator.cc +1262 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/python/python_generator.h +166 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/python/python_plugin_unittest.cc +118 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/subprocess.cc +463 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/subprocess.h +108 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/test_plugin.cc +51 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/zip_output_unittest.sh +91 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/zip_writer.cc +218 -0
- data/ext/protoc/protobuf/src/google/protobuf/compiler/zip_writer.h +93 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor.cc +5420 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor.h +1691 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor.pb.cc +9135 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor.pb.h +6761 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor.proto +687 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor_database.cc +543 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor_database.h +369 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor_database_unittest.cc +748 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor_pb2_test.py +54 -0
- data/ext/protoc/protobuf/src/google/protobuf/descriptor_unittest.cc +5501 -0
- data/ext/protoc/protobuf/src/google/protobuf/dynamic_message.cc +764 -0
- data/ext/protoc/protobuf/src/google/protobuf/dynamic_message.h +148 -0
- data/ext/protoc/protobuf/src/google/protobuf/dynamic_message_unittest.cc +230 -0
- data/ext/protoc/protobuf/src/google/protobuf/extension_set.cc +1663 -0
- data/ext/protoc/protobuf/src/google/protobuf/extension_set.h +1234 -0
- data/ext/protoc/protobuf/src/google/protobuf/extension_set_heavy.cc +734 -0
- data/ext/protoc/protobuf/src/google/protobuf/extension_set_unittest.cc +1095 -0
- data/ext/protoc/protobuf/src/google/protobuf/generated_enum_reflection.h +91 -0
- data/ext/protoc/protobuf/src/google/protobuf/generated_message_reflection.cc +1683 -0
- data/ext/protoc/protobuf/src/google/protobuf/generated_message_reflection.h +504 -0
- data/ext/protoc/protobuf/src/google/protobuf/generated_message_reflection_unittest.cc +795 -0
- data/ext/protoc/protobuf/src/google/protobuf/generated_message_util.cc +65 -0
- data/ext/protoc/protobuf/src/google/protobuf/generated_message_util.h +113 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/coded_stream.cc +914 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/coded_stream.h +1220 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/coded_stream_inl.h +69 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/coded_stream_unittest.cc +1378 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/gzip_stream.cc +326 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/gzip_stream.h +209 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/gzip_stream_unittest.sh +44 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/package_info.h +54 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/printer.cc +198 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/printer.h +136 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/printer_unittest.cc +285 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/strtod.cc +113 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/strtod.h +50 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/tokenizer.cc +1127 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/tokenizer.h +402 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/tokenizer_unittest.cc +999 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/zero_copy_stream.cc +57 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/zero_copy_stream.h +248 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/zero_copy_stream_impl.cc +473 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/zero_copy_stream_impl.h +358 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/zero_copy_stream_impl_lite.cc +405 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/zero_copy_stream_impl_lite.h +354 -0
- data/ext/protoc/protobuf/src/google/protobuf/io/zero_copy_stream_unittest.cc +965 -0
- data/ext/protoc/protobuf/src/google/protobuf/lite_unittest.cc +350 -0
- data/ext/protoc/protobuf/src/google/protobuf/message.cc +358 -0
- data/ext/protoc/protobuf/src/google/protobuf/message.h +866 -0
- data/ext/protoc/protobuf/src/google/protobuf/message_lite.cc +335 -0
- data/ext/protoc/protobuf/src/google/protobuf/message_lite.h +247 -0
- data/ext/protoc/protobuf/src/google/protobuf/message_unittest.cc +427 -0
- data/ext/protoc/protobuf/src/google/protobuf/package_info.h +64 -0
- data/ext/protoc/protobuf/src/google/protobuf/reflection_ops.cc +269 -0
- data/ext/protoc/protobuf/src/google/protobuf/reflection_ops.h +81 -0
- data/ext/protoc/protobuf/src/google/protobuf/reflection_ops_unittest.cc +475 -0
- data/ext/protoc/protobuf/src/google/protobuf/repeated_field.cc +87 -0
- data/ext/protoc/protobuf/src/google/protobuf/repeated_field.h +1603 -0
- data/ext/protoc/protobuf/src/google/protobuf/repeated_field_reflection_unittest.cc +195 -0
- data/ext/protoc/protobuf/src/google/protobuf/repeated_field_unittest.cc +1442 -0
- data/ext/protoc/protobuf/src/google/protobuf/service.cc +46 -0
- data/ext/protoc/protobuf/src/google/protobuf/service.h +291 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops.h +227 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_arm64_gcc.h +325 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_arm_gcc.h +151 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_arm_qnx.h +146 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_atomicword_compat.h +122 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_generic_gcc.h +137 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_macosx.h +225 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_mips_gcc.h +313 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_pnacl.h +73 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_solaris.h +188 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_tsan.h +219 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_gcc.cc +137 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_gcc.h +293 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_msvc.cc +112 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/atomicops_internals_x86_msvc.h +150 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/common.cc +395 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/common.h +1226 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/common_unittest.cc +357 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/hash.h +232 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/map_util.h +771 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/once.cc +99 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/once.h +166 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/once_unittest.cc +253 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/platform_macros.h +103 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/shared_ptr.h +470 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/stl_util.h +121 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/stringprintf.cc +175 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/stringprintf.h +76 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/stringprintf_unittest.cc +152 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/structurally_valid.cc +536 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/structurally_valid_unittest.cc +40 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/strutil.cc +1279 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/strutil.h +562 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/strutil_unittest.cc +73 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/substitute.cc +134 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/substitute.h +170 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/template_util.h +138 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/template_util_unittest.cc +130 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/type_traits.h +336 -0
- data/ext/protoc/protobuf/src/google/protobuf/stubs/type_traits_unittest.cc +628 -0
- data/ext/protoc/protobuf/src/google/protobuf/test_util.cc +3345 -0
- data/ext/protoc/protobuf/src/google/protobuf/test_util.h +215 -0
- data/ext/protoc/protobuf/src/google/protobuf/test_util_lite.cc +1585 -0
- data/ext/protoc/protobuf/src/google/protobuf/test_util_lite.h +101 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/bad_utf8_string +1 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/golden_message +0 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/golden_message_oneof_implemented +0 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/golden_packed_fields_message +0 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/text_format_unittest_data.txt +134 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/text_format_unittest_data_oneof_implemented.txt +129 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/text_format_unittest_data_pointy.txt +134 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/text_format_unittest_data_pointy_oneof.txt +129 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/text_format_unittest_extensions_data.txt +134 -0
- data/ext/protoc/protobuf/src/google/protobuf/testdata/text_format_unittest_extensions_data_pointy.txt +134 -0
- data/ext/protoc/protobuf/src/google/protobuf/testing/file.cc +194 -0
- data/ext/protoc/protobuf/src/google/protobuf/testing/file.h +97 -0
- data/ext/protoc/protobuf/src/google/protobuf/testing/googletest.cc +255 -0
- data/ext/protoc/protobuf/src/google/protobuf/testing/googletest.h +102 -0
- data/ext/protoc/protobuf/src/google/protobuf/testing/zcgunzip.cc +73 -0
- data/ext/protoc/protobuf/src/google/protobuf/testing/zcgzip.cc +79 -0
- data/ext/protoc/protobuf/src/google/protobuf/text_format.cc +1746 -0
- data/ext/protoc/protobuf/src/google/protobuf/text_format.h +473 -0
- data/ext/protoc/protobuf/src/google/protobuf/text_format_unittest.cc +1479 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest.proto +861 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_custom_options.proto +393 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_embed_optimize_for.proto +50 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_empty.proto +37 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_enormous_descriptor.proto +1046 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_import.proto +64 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_import_lite.proto +51 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_import_public.proto +40 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_import_public_lite.proto +42 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_lite.proto +384 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_lite_imports_nonlite.proto +43 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_mset.proto +83 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_no_generic_services.proto +53 -0
- data/ext/protoc/protobuf/src/google/protobuf/unittest_optimize_for.proto +66 -0
- data/ext/protoc/protobuf/src/google/protobuf/unknown_field_set.cc +265 -0
- data/ext/protoc/protobuf/src/google/protobuf/unknown_field_set.h +318 -0
- data/ext/protoc/protobuf/src/google/protobuf/unknown_field_set_unittest.cc +599 -0
- data/ext/protoc/protobuf/src/google/protobuf/wire_format.cc +1101 -0
- data/ext/protoc/protobuf/src/google/protobuf/wire_format.h +336 -0
- data/ext/protoc/protobuf/src/google/protobuf/wire_format_lite.cc +471 -0
- data/ext/protoc/protobuf/src/google/protobuf/wire_format_lite.h +661 -0
- data/ext/protoc/protobuf/src/google/protobuf/wire_format_lite_inl.h +860 -0
- data/ext/protoc/protobuf/src/google/protobuf/wire_format_unittest.cc +1120 -0
- data/ext/protoc/protobuf/src/solaris/libstdc++.la +51 -0
- data/ext/protoc/protobuf/vsprojects/config.h +29 -0
- data/ext/protoc/protobuf/vsprojects/convert2008to2005.sh +20 -0
- data/ext/protoc/protobuf/vsprojects/extract_includes.bat +50 -0
- data/ext/protoc/protobuf/vsprojects/libprotobuf-lite.vcproj +302 -0
- data/ext/protoc/protobuf/vsprojects/libprotobuf.vcproj +470 -0
- data/ext/protoc/protobuf/vsprojects/libprotoc.vcproj +466 -0
- data/ext/protoc/protobuf/vsprojects/lite-test.vcproj +305 -0
- data/ext/protoc/protobuf/vsprojects/protobuf.sln +92 -0
- data/ext/protoc/protobuf/vsprojects/protoc.vcproj +192 -0
- data/ext/protoc/protobuf/vsprojects/readme.txt +114 -0
- data/ext/protoc/protobuf/vsprojects/test_plugin.vcproj +209 -0
- data/ext/protoc/protobuf/vsprojects/tests.vcproj +681 -0
- data/lib/protoc/version.rb +1 -1
- metadata +480 -3
@@ -0,0 +1,113 @@
|
|
1
|
+
// Protocol Buffers - Google's data interchange format
|
2
|
+
// Copyright 2008 Google Inc. All rights reserved.
|
3
|
+
// https://developers.google.com/protocol-buffers/
|
4
|
+
//
|
5
|
+
// Redistribution and use in source and binary forms, with or without
|
6
|
+
// modification, are permitted provided that the following conditions are
|
7
|
+
// met:
|
8
|
+
//
|
9
|
+
// * Redistributions of source code must retain the above copyright
|
10
|
+
// notice, this list of conditions and the following disclaimer.
|
11
|
+
// * Redistributions in binary form must reproduce the above
|
12
|
+
// copyright notice, this list of conditions and the following disclaimer
|
13
|
+
// in the documentation and/or other materials provided with the
|
14
|
+
// distribution.
|
15
|
+
// * Neither the name of Google Inc. nor the names of its
|
16
|
+
// contributors may be used to endorse or promote products derived from
|
17
|
+
// this software without specific prior written permission.
|
18
|
+
//
|
19
|
+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
+
|
31
|
+
#include <google/protobuf/io/strtod.h>
|
32
|
+
|
33
|
+
#include <cstdio>
|
34
|
+
#include <cstring>
|
35
|
+
#include <string>
|
36
|
+
|
37
|
+
#include <google/protobuf/stubs/common.h>
|
38
|
+
|
39
|
+
namespace google {
|
40
|
+
namespace protobuf {
|
41
|
+
namespace io {
|
42
|
+
|
43
|
+
// ----------------------------------------------------------------------
|
44
|
+
// NoLocaleStrtod()
|
45
|
+
// This code will make you cry.
|
46
|
+
// ----------------------------------------------------------------------
|
47
|
+
|
48
|
+
namespace {
|
49
|
+
|
50
|
+
// Returns a string identical to *input except that the character pointed to
|
51
|
+
// by radix_pos (which should be '.') is replaced with the locale-specific
|
52
|
+
// radix character.
|
53
|
+
string LocalizeRadix(const char* input, const char* radix_pos) {
|
54
|
+
// Determine the locale-specific radix character by calling sprintf() to
|
55
|
+
// print the number 1.5, then stripping off the digits. As far as I can
|
56
|
+
// tell, this is the only portable, thread-safe way to get the C library
|
57
|
+
// to divuldge the locale's radix character. No, localeconv() is NOT
|
58
|
+
// thread-safe.
|
59
|
+
char temp[16];
|
60
|
+
int size = sprintf(temp, "%.1f", 1.5);
|
61
|
+
GOOGLE_CHECK_EQ(temp[0], '1');
|
62
|
+
GOOGLE_CHECK_EQ(temp[size-1], '5');
|
63
|
+
GOOGLE_CHECK_LE(size, 6);
|
64
|
+
|
65
|
+
// Now replace the '.' in the input with it.
|
66
|
+
string result;
|
67
|
+
result.reserve(strlen(input) + size - 3);
|
68
|
+
result.append(input, radix_pos);
|
69
|
+
result.append(temp + 1, size - 2);
|
70
|
+
result.append(radix_pos + 1);
|
71
|
+
return result;
|
72
|
+
}
|
73
|
+
|
74
|
+
} // namespace
|
75
|
+
|
76
|
+
double NoLocaleStrtod(const char* text, char** original_endptr) {
|
77
|
+
// We cannot simply set the locale to "C" temporarily with setlocale()
|
78
|
+
// as this is not thread-safe. Instead, we try to parse in the current
|
79
|
+
// locale first. If parsing stops at a '.' character, then this is a
|
80
|
+
// pretty good hint that we're actually in some other locale in which
|
81
|
+
// '.' is not the radix character.
|
82
|
+
|
83
|
+
char* temp_endptr;
|
84
|
+
double result = strtod(text, &temp_endptr);
|
85
|
+
if (original_endptr != NULL) *original_endptr = temp_endptr;
|
86
|
+
if (*temp_endptr != '.') return result;
|
87
|
+
|
88
|
+
// Parsing halted on a '.'. Perhaps we're in a different locale? Let's
|
89
|
+
// try to replace the '.' with a locale-specific radix character and
|
90
|
+
// try again.
|
91
|
+
string localized = LocalizeRadix(text, temp_endptr);
|
92
|
+
const char* localized_cstr = localized.c_str();
|
93
|
+
char* localized_endptr;
|
94
|
+
result = strtod(localized_cstr, &localized_endptr);
|
95
|
+
if ((localized_endptr - localized_cstr) >
|
96
|
+
(temp_endptr - text)) {
|
97
|
+
// This attempt got further, so replacing the decimal must have helped.
|
98
|
+
// Update original_endptr to point at the right location.
|
99
|
+
if (original_endptr != NULL) {
|
100
|
+
// size_diff is non-zero if the localized radix has multiple bytes.
|
101
|
+
int size_diff = localized.size() - strlen(text);
|
102
|
+
// const_cast is necessary to match the strtod() interface.
|
103
|
+
*original_endptr = const_cast<char*>(
|
104
|
+
text + (localized_endptr - localized_cstr - size_diff));
|
105
|
+
}
|
106
|
+
}
|
107
|
+
|
108
|
+
return result;
|
109
|
+
}
|
110
|
+
|
111
|
+
} // namespace io
|
112
|
+
} // namespace protobuf
|
113
|
+
} // namespace google
|
@@ -0,0 +1,50 @@
|
|
1
|
+
// Protocol Buffers - Google's data interchange format
|
2
|
+
// Copyright 2008 Google Inc. All rights reserved.
|
3
|
+
// https://developers.google.com/protocol-buffers/
|
4
|
+
//
|
5
|
+
// Redistribution and use in source and binary forms, with or without
|
6
|
+
// modification, are permitted provided that the following conditions are
|
7
|
+
// met:
|
8
|
+
//
|
9
|
+
// * Redistributions of source code must retain the above copyright
|
10
|
+
// notice, this list of conditions and the following disclaimer.
|
11
|
+
// * Redistributions in binary form must reproduce the above
|
12
|
+
// copyright notice, this list of conditions and the following disclaimer
|
13
|
+
// in the documentation and/or other materials provided with the
|
14
|
+
// distribution.
|
15
|
+
// * Neither the name of Google Inc. nor the names of its
|
16
|
+
// contributors may be used to endorse or promote products derived from
|
17
|
+
// this software without specific prior written permission.
|
18
|
+
//
|
19
|
+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
+
|
31
|
+
// A locale-independent version of strtod(), used to parse floating
|
32
|
+
// point default values in .proto files, where the decimal separator
|
33
|
+
// is always a dot.
|
34
|
+
|
35
|
+
#ifndef GOOGLE_PROTOBUF_IO_STRTOD_H__
|
36
|
+
#define GOOGLE_PROTOBUF_IO_STRTOD_H__
|
37
|
+
|
38
|
+
namespace google {
|
39
|
+
namespace protobuf {
|
40
|
+
namespace io {
|
41
|
+
|
42
|
+
// A locale-independent version of the standard strtod(), which always
|
43
|
+
// uses a dot as the decimal separator.
|
44
|
+
double NoLocaleStrtod(const char* str, char** endptr);
|
45
|
+
|
46
|
+
} // namespace io
|
47
|
+
} // namespace protobuf
|
48
|
+
|
49
|
+
} // namespace google
|
50
|
+
#endif // GOOGLE_PROTOBUF_IO_STRTOD_H__
|
@@ -0,0 +1,1127 @@
|
|
1
|
+
// Protocol Buffers - Google's data interchange format
|
2
|
+
// Copyright 2008 Google Inc. All rights reserved.
|
3
|
+
// https://developers.google.com/protocol-buffers/
|
4
|
+
//
|
5
|
+
// Redistribution and use in source and binary forms, with or without
|
6
|
+
// modification, are permitted provided that the following conditions are
|
7
|
+
// met:
|
8
|
+
//
|
9
|
+
// * Redistributions of source code must retain the above copyright
|
10
|
+
// notice, this list of conditions and the following disclaimer.
|
11
|
+
// * Redistributions in binary form must reproduce the above
|
12
|
+
// copyright notice, this list of conditions and the following disclaimer
|
13
|
+
// in the documentation and/or other materials provided with the
|
14
|
+
// distribution.
|
15
|
+
// * Neither the name of Google Inc. nor the names of its
|
16
|
+
// contributors may be used to endorse or promote products derived from
|
17
|
+
// this software without specific prior written permission.
|
18
|
+
//
|
19
|
+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20
|
+
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21
|
+
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22
|
+
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23
|
+
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24
|
+
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25
|
+
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26
|
+
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27
|
+
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28
|
+
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30
|
+
|
31
|
+
// Author: kenton@google.com (Kenton Varda)
|
32
|
+
// Based on original Protocol Buffers design by
|
33
|
+
// Sanjay Ghemawat, Jeff Dean, and others.
|
34
|
+
//
|
35
|
+
// Here we have a hand-written lexer. At first you might ask yourself,
|
36
|
+
// "Hand-written text processing? Is Kenton crazy?!" Well, first of all,
|
37
|
+
// yes I am crazy, but that's beside the point. There are actually reasons
|
38
|
+
// why I ended up writing this this way.
|
39
|
+
//
|
40
|
+
// The traditional approach to lexing is to use lex to generate a lexer for
|
41
|
+
// you. Unfortunately, lex's output is ridiculously ugly and difficult to
|
42
|
+
// integrate cleanly with C++ code, especially abstract code or code meant
|
43
|
+
// as a library. Better parser-generators exist but would add dependencies
|
44
|
+
// which most users won't already have, which we'd like to avoid. (GNU flex
|
45
|
+
// has a C++ output option, but it's still ridiculously ugly, non-abstract,
|
46
|
+
// and not library-friendly.)
|
47
|
+
//
|
48
|
+
// The next approach that any good software engineer should look at is to
|
49
|
+
// use regular expressions. And, indeed, I did. I have code which
|
50
|
+
// implements this same class using regular expressions. It's about 200
|
51
|
+
// lines shorter. However:
|
52
|
+
// - Rather than error messages telling you "This string has an invalid
|
53
|
+
// escape sequence at line 5, column 45", you get error messages like
|
54
|
+
// "Parse error on line 5". Giving more precise errors requires adding
|
55
|
+
// a lot of code that ends up basically as complex as the hand-coded
|
56
|
+
// version anyway.
|
57
|
+
// - The regular expression to match a string literal looks like this:
|
58
|
+
// kString = new RE("(\"([^\"\\\\]|" // non-escaped
|
59
|
+
// "\\\\[abfnrtv?\"'\\\\0-7]|" // normal escape
|
60
|
+
// "\\\\x[0-9a-fA-F])*\"|" // hex escape
|
61
|
+
// "\'([^\'\\\\]|" // Also support single-quotes.
|
62
|
+
// "\\\\[abfnrtv?\"'\\\\0-7]|"
|
63
|
+
// "\\\\x[0-9a-fA-F])*\')");
|
64
|
+
// Verifying the correctness of this line noise is actually harder than
|
65
|
+
// verifying the correctness of ConsumeString(), defined below. I'm not
|
66
|
+
// even confident that the above is correct, after staring at it for some
|
67
|
+
// time.
|
68
|
+
// - PCRE is fast, but there's still more overhead involved than the code
|
69
|
+
// below.
|
70
|
+
// - Sadly, regular expressions are not part of the C standard library, so
|
71
|
+
// using them would require depending on some other library. For the
|
72
|
+
// open source release, this could be really annoying. Nobody likes
|
73
|
+
// downloading one piece of software just to find that they need to
|
74
|
+
// download something else to make it work, and in all likelihood
|
75
|
+
// people downloading Protocol Buffers will already be doing so just
|
76
|
+
// to make something else work. We could include a copy of PCRE with
|
77
|
+
// our code, but that obligates us to keep it up-to-date and just seems
|
78
|
+
// like a big waste just to save 200 lines of code.
|
79
|
+
//
|
80
|
+
// On a similar but unrelated note, I'm even scared to use ctype.h.
|
81
|
+
// Apparently functions like isalpha() are locale-dependent. So, if we used
|
82
|
+
// that, then if this code is being called from some program that doesn't
|
83
|
+
// have its locale set to "C", it would behave strangely. We can't just set
|
84
|
+
// the locale to "C" ourselves since we might break the calling program that
|
85
|
+
// way, particularly if it is multi-threaded. WTF? Someone please let me
|
86
|
+
// (Kenton) know if I'm missing something here...
|
87
|
+
//
|
88
|
+
// I'd love to hear about other alternatives, though, as this code isn't
|
89
|
+
// exactly pretty.
|
90
|
+
|
91
|
+
#include <google/protobuf/io/tokenizer.h>
|
92
|
+
#include <google/protobuf/stubs/common.h>
|
93
|
+
#include <google/protobuf/stubs/stringprintf.h>
|
94
|
+
#include <google/protobuf/io/strtod.h>
|
95
|
+
#include <google/protobuf/io/zero_copy_stream.h>
|
96
|
+
#include <google/protobuf/stubs/strutil.h>
|
97
|
+
#include <google/protobuf/stubs/stl_util.h>
|
98
|
+
|
99
|
+
namespace google {
|
100
|
+
namespace protobuf {
|
101
|
+
namespace io {
|
102
|
+
namespace {
|
103
|
+
|
104
|
+
// As mentioned above, I don't trust ctype.h due to the presence of "locales".
|
105
|
+
// So, I have written replacement functions here. Someone please smack me if
|
106
|
+
// this is a bad idea or if there is some way around this.
|
107
|
+
//
|
108
|
+
// These "character classes" are designed to be used in template methods.
|
109
|
+
// For instance, Tokenizer::ConsumeZeroOrMore<Whitespace>() will eat
|
110
|
+
// whitespace.
|
111
|
+
|
112
|
+
// Note: No class is allowed to contain '\0', since this is used to mark end-
|
113
|
+
// of-input and is handled specially.
|
114
|
+
|
115
|
+
#define CHARACTER_CLASS(NAME, EXPRESSION) \
|
116
|
+
class NAME { \
|
117
|
+
public: \
|
118
|
+
static inline bool InClass(char c) { \
|
119
|
+
return EXPRESSION; \
|
120
|
+
} \
|
121
|
+
}
|
122
|
+
|
123
|
+
CHARACTER_CLASS(Whitespace, c == ' ' || c == '\n' || c == '\t' ||
|
124
|
+
c == '\r' || c == '\v' || c == '\f');
|
125
|
+
CHARACTER_CLASS(WhitespaceNoNewline, c == ' ' || c == '\t' ||
|
126
|
+
c == '\r' || c == '\v' || c == '\f');
|
127
|
+
|
128
|
+
CHARACTER_CLASS(Unprintable, c < ' ' && c > '\0');
|
129
|
+
|
130
|
+
CHARACTER_CLASS(Digit, '0' <= c && c <= '9');
|
131
|
+
CHARACTER_CLASS(OctalDigit, '0' <= c && c <= '7');
|
132
|
+
CHARACTER_CLASS(HexDigit, ('0' <= c && c <= '9') ||
|
133
|
+
('a' <= c && c <= 'f') ||
|
134
|
+
('A' <= c && c <= 'F'));
|
135
|
+
|
136
|
+
CHARACTER_CLASS(Letter, ('a' <= c && c <= 'z') ||
|
137
|
+
('A' <= c && c <= 'Z') ||
|
138
|
+
(c == '_'));
|
139
|
+
|
140
|
+
CHARACTER_CLASS(Alphanumeric, ('a' <= c && c <= 'z') ||
|
141
|
+
('A' <= c && c <= 'Z') ||
|
142
|
+
('0' <= c && c <= '9') ||
|
143
|
+
(c == '_'));
|
144
|
+
|
145
|
+
CHARACTER_CLASS(Escape, c == 'a' || c == 'b' || c == 'f' || c == 'n' ||
|
146
|
+
c == 'r' || c == 't' || c == 'v' || c == '\\' ||
|
147
|
+
c == '?' || c == '\'' || c == '\"');
|
148
|
+
|
149
|
+
#undef CHARACTER_CLASS
|
150
|
+
|
151
|
+
// Given a char, interpret it as a numeric digit and return its value.
|
152
|
+
// This supports any number base up to 36.
|
153
|
+
inline int DigitValue(char digit) {
|
154
|
+
if ('0' <= digit && digit <= '9') return digit - '0';
|
155
|
+
if ('a' <= digit && digit <= 'z') return digit - 'a' + 10;
|
156
|
+
if ('A' <= digit && digit <= 'Z') return digit - 'A' + 10;
|
157
|
+
return -1;
|
158
|
+
}
|
159
|
+
|
160
|
+
// Inline because it's only used in one place.
|
161
|
+
inline char TranslateEscape(char c) {
|
162
|
+
switch (c) {
|
163
|
+
case 'a': return '\a';
|
164
|
+
case 'b': return '\b';
|
165
|
+
case 'f': return '\f';
|
166
|
+
case 'n': return '\n';
|
167
|
+
case 'r': return '\r';
|
168
|
+
case 't': return '\t';
|
169
|
+
case 'v': return '\v';
|
170
|
+
case '\\': return '\\';
|
171
|
+
case '?': return '\?'; // Trigraphs = :(
|
172
|
+
case '\'': return '\'';
|
173
|
+
case '"': return '\"';
|
174
|
+
|
175
|
+
// We expect escape sequences to have been validated separately.
|
176
|
+
default: return '?';
|
177
|
+
}
|
178
|
+
}
|
179
|
+
|
180
|
+
} // anonymous namespace
|
181
|
+
|
182
|
+
ErrorCollector::~ErrorCollector() {}
|
183
|
+
|
184
|
+
// ===================================================================
|
185
|
+
|
186
|
+
Tokenizer::Tokenizer(ZeroCopyInputStream* input,
|
187
|
+
ErrorCollector* error_collector)
|
188
|
+
: input_(input),
|
189
|
+
error_collector_(error_collector),
|
190
|
+
buffer_(NULL),
|
191
|
+
buffer_size_(0),
|
192
|
+
buffer_pos_(0),
|
193
|
+
read_error_(false),
|
194
|
+
line_(0),
|
195
|
+
column_(0),
|
196
|
+
record_target_(NULL),
|
197
|
+
record_start_(-1),
|
198
|
+
allow_f_after_float_(false),
|
199
|
+
comment_style_(CPP_COMMENT_STYLE),
|
200
|
+
require_space_after_number_(true),
|
201
|
+
allow_multiline_strings_(false) {
|
202
|
+
|
203
|
+
current_.line = 0;
|
204
|
+
current_.column = 0;
|
205
|
+
current_.end_column = 0;
|
206
|
+
current_.type = TYPE_START;
|
207
|
+
|
208
|
+
Refresh();
|
209
|
+
}
|
210
|
+
|
211
|
+
Tokenizer::~Tokenizer() {
|
212
|
+
// If we had any buffer left unread, return it to the underlying stream
|
213
|
+
// so that someone else can read it.
|
214
|
+
if (buffer_size_ > buffer_pos_) {
|
215
|
+
input_->BackUp(buffer_size_ - buffer_pos_);
|
216
|
+
}
|
217
|
+
}
|
218
|
+
|
219
|
+
// -------------------------------------------------------------------
|
220
|
+
// Internal helpers.
|
221
|
+
|
222
|
+
void Tokenizer::NextChar() {
|
223
|
+
// Update our line and column counters based on the character being
|
224
|
+
// consumed.
|
225
|
+
if (current_char_ == '\n') {
|
226
|
+
++line_;
|
227
|
+
column_ = 0;
|
228
|
+
} else if (current_char_ == '\t') {
|
229
|
+
column_ += kTabWidth - column_ % kTabWidth;
|
230
|
+
} else {
|
231
|
+
++column_;
|
232
|
+
}
|
233
|
+
|
234
|
+
// Advance to the next character.
|
235
|
+
++buffer_pos_;
|
236
|
+
if (buffer_pos_ < buffer_size_) {
|
237
|
+
current_char_ = buffer_[buffer_pos_];
|
238
|
+
} else {
|
239
|
+
Refresh();
|
240
|
+
}
|
241
|
+
}
|
242
|
+
|
243
|
+
void Tokenizer::Refresh() {
|
244
|
+
if (read_error_) {
|
245
|
+
current_char_ = '\0';
|
246
|
+
return;
|
247
|
+
}
|
248
|
+
|
249
|
+
// If we're in a token, append the rest of the buffer to it.
|
250
|
+
if (record_target_ != NULL && record_start_ < buffer_size_) {
|
251
|
+
record_target_->append(buffer_ + record_start_, buffer_size_ - record_start_);
|
252
|
+
record_start_ = 0;
|
253
|
+
}
|
254
|
+
|
255
|
+
const void* data = NULL;
|
256
|
+
buffer_ = NULL;
|
257
|
+
buffer_pos_ = 0;
|
258
|
+
do {
|
259
|
+
if (!input_->Next(&data, &buffer_size_)) {
|
260
|
+
// end of stream (or read error)
|
261
|
+
buffer_size_ = 0;
|
262
|
+
read_error_ = true;
|
263
|
+
current_char_ = '\0';
|
264
|
+
return;
|
265
|
+
}
|
266
|
+
} while (buffer_size_ == 0);
|
267
|
+
|
268
|
+
buffer_ = static_cast<const char*>(data);
|
269
|
+
|
270
|
+
current_char_ = buffer_[0];
|
271
|
+
}
|
272
|
+
|
273
|
+
inline void Tokenizer::RecordTo(string* target) {
|
274
|
+
record_target_ = target;
|
275
|
+
record_start_ = buffer_pos_;
|
276
|
+
}
|
277
|
+
|
278
|
+
inline void Tokenizer::StopRecording() {
|
279
|
+
// Note: The if() is necessary because some STL implementations crash when
|
280
|
+
// you call string::append(NULL, 0), presumably because they are trying to
|
281
|
+
// be helpful by detecting the NULL pointer, even though there's nothing
|
282
|
+
// wrong with reading zero bytes from NULL.
|
283
|
+
if (buffer_pos_ != record_start_) {
|
284
|
+
record_target_->append(buffer_ + record_start_, buffer_pos_ - record_start_);
|
285
|
+
}
|
286
|
+
record_target_ = NULL;
|
287
|
+
record_start_ = -1;
|
288
|
+
}
|
289
|
+
|
290
|
+
inline void Tokenizer::StartToken() {
|
291
|
+
current_.type = TYPE_START; // Just for the sake of initializing it.
|
292
|
+
current_.text.clear();
|
293
|
+
current_.line = line_;
|
294
|
+
current_.column = column_;
|
295
|
+
RecordTo(¤t_.text);
|
296
|
+
}
|
297
|
+
|
298
|
+
inline void Tokenizer::EndToken() {
|
299
|
+
StopRecording();
|
300
|
+
current_.end_column = column_;
|
301
|
+
}
|
302
|
+
|
303
|
+
// -------------------------------------------------------------------
|
304
|
+
// Helper methods that consume characters.
|
305
|
+
|
306
|
+
template<typename CharacterClass>
|
307
|
+
inline bool Tokenizer::LookingAt() {
|
308
|
+
return CharacterClass::InClass(current_char_);
|
309
|
+
}
|
310
|
+
|
311
|
+
template<typename CharacterClass>
|
312
|
+
inline bool Tokenizer::TryConsumeOne() {
|
313
|
+
if (CharacterClass::InClass(current_char_)) {
|
314
|
+
NextChar();
|
315
|
+
return true;
|
316
|
+
} else {
|
317
|
+
return false;
|
318
|
+
}
|
319
|
+
}
|
320
|
+
|
321
|
+
inline bool Tokenizer::TryConsume(char c) {
|
322
|
+
if (current_char_ == c) {
|
323
|
+
NextChar();
|
324
|
+
return true;
|
325
|
+
} else {
|
326
|
+
return false;
|
327
|
+
}
|
328
|
+
}
|
329
|
+
|
330
|
+
template<typename CharacterClass>
|
331
|
+
inline void Tokenizer::ConsumeZeroOrMore() {
|
332
|
+
while (CharacterClass::InClass(current_char_)) {
|
333
|
+
NextChar();
|
334
|
+
}
|
335
|
+
}
|
336
|
+
|
337
|
+
template<typename CharacterClass>
|
338
|
+
inline void Tokenizer::ConsumeOneOrMore(const char* error) {
|
339
|
+
if (!CharacterClass::InClass(current_char_)) {
|
340
|
+
AddError(error);
|
341
|
+
} else {
|
342
|
+
do {
|
343
|
+
NextChar();
|
344
|
+
} while (CharacterClass::InClass(current_char_));
|
345
|
+
}
|
346
|
+
}
|
347
|
+
|
348
|
+
// -------------------------------------------------------------------
|
349
|
+
// Methods that read whole patterns matching certain kinds of tokens
|
350
|
+
// or comments.
|
351
|
+
|
352
|
+
void Tokenizer::ConsumeString(char delimiter) {
|
353
|
+
while (true) {
|
354
|
+
switch (current_char_) {
|
355
|
+
case '\0':
|
356
|
+
AddError("Unexpected end of string.");
|
357
|
+
return;
|
358
|
+
|
359
|
+
case '\n': {
|
360
|
+
if (!allow_multiline_strings_) {
|
361
|
+
AddError("String literals cannot cross line boundaries.");
|
362
|
+
return;
|
363
|
+
}
|
364
|
+
NextChar();
|
365
|
+
break;
|
366
|
+
}
|
367
|
+
|
368
|
+
case '\\': {
|
369
|
+
// An escape sequence.
|
370
|
+
NextChar();
|
371
|
+
if (TryConsumeOne<Escape>()) {
|
372
|
+
// Valid escape sequence.
|
373
|
+
} else if (TryConsumeOne<OctalDigit>()) {
|
374
|
+
// Possibly followed by two more octal digits, but these will
|
375
|
+
// just be consumed by the main loop anyway so we don't need
|
376
|
+
// to do so explicitly here.
|
377
|
+
} else if (TryConsume('x') || TryConsume('X')) {
|
378
|
+
if (!TryConsumeOne<HexDigit>()) {
|
379
|
+
AddError("Expected hex digits for escape sequence.");
|
380
|
+
}
|
381
|
+
// Possibly followed by another hex digit, but again we don't care.
|
382
|
+
} else if (TryConsume('u')) {
|
383
|
+
if (!TryConsumeOne<HexDigit>() ||
|
384
|
+
!TryConsumeOne<HexDigit>() ||
|
385
|
+
!TryConsumeOne<HexDigit>() ||
|
386
|
+
!TryConsumeOne<HexDigit>()) {
|
387
|
+
AddError("Expected four hex digits for \\u escape sequence.");
|
388
|
+
}
|
389
|
+
} else if (TryConsume('U')) {
|
390
|
+
// We expect 8 hex digits; but only the range up to 0x10ffff is
|
391
|
+
// legal.
|
392
|
+
if (!TryConsume('0') ||
|
393
|
+
!TryConsume('0') ||
|
394
|
+
!(TryConsume('0') || TryConsume('1')) ||
|
395
|
+
!TryConsumeOne<HexDigit>() ||
|
396
|
+
!TryConsumeOne<HexDigit>() ||
|
397
|
+
!TryConsumeOne<HexDigit>() ||
|
398
|
+
!TryConsumeOne<HexDigit>() ||
|
399
|
+
!TryConsumeOne<HexDigit>()) {
|
400
|
+
AddError("Expected eight hex digits up to 10ffff for \\U escape "
|
401
|
+
"sequence");
|
402
|
+
}
|
403
|
+
} else {
|
404
|
+
AddError("Invalid escape sequence in string literal.");
|
405
|
+
}
|
406
|
+
break;
|
407
|
+
}
|
408
|
+
|
409
|
+
default: {
|
410
|
+
if (current_char_ == delimiter) {
|
411
|
+
NextChar();
|
412
|
+
return;
|
413
|
+
}
|
414
|
+
NextChar();
|
415
|
+
break;
|
416
|
+
}
|
417
|
+
}
|
418
|
+
}
|
419
|
+
}
|
420
|
+
|
421
|
+
Tokenizer::TokenType Tokenizer::ConsumeNumber(bool started_with_zero,
|
422
|
+
bool started_with_dot) {
|
423
|
+
bool is_float = false;
|
424
|
+
|
425
|
+
if (started_with_zero && (TryConsume('x') || TryConsume('X'))) {
|
426
|
+
// A hex number (started with "0x").
|
427
|
+
ConsumeOneOrMore<HexDigit>("\"0x\" must be followed by hex digits.");
|
428
|
+
|
429
|
+
} else if (started_with_zero && LookingAt<Digit>()) {
|
430
|
+
// An octal number (had a leading zero).
|
431
|
+
ConsumeZeroOrMore<OctalDigit>();
|
432
|
+
if (LookingAt<Digit>()) {
|
433
|
+
AddError("Numbers starting with leading zero must be in octal.");
|
434
|
+
ConsumeZeroOrMore<Digit>();
|
435
|
+
}
|
436
|
+
|
437
|
+
} else {
|
438
|
+
// A decimal number.
|
439
|
+
if (started_with_dot) {
|
440
|
+
is_float = true;
|
441
|
+
ConsumeZeroOrMore<Digit>();
|
442
|
+
} else {
|
443
|
+
ConsumeZeroOrMore<Digit>();
|
444
|
+
|
445
|
+
if (TryConsume('.')) {
|
446
|
+
is_float = true;
|
447
|
+
ConsumeZeroOrMore<Digit>();
|
448
|
+
}
|
449
|
+
}
|
450
|
+
|
451
|
+
if (TryConsume('e') || TryConsume('E')) {
|
452
|
+
is_float = true;
|
453
|
+
TryConsume('-') || TryConsume('+');
|
454
|
+
ConsumeOneOrMore<Digit>("\"e\" must be followed by exponent.");
|
455
|
+
}
|
456
|
+
|
457
|
+
if (allow_f_after_float_ && (TryConsume('f') || TryConsume('F'))) {
|
458
|
+
is_float = true;
|
459
|
+
}
|
460
|
+
}
|
461
|
+
|
462
|
+
if (LookingAt<Letter>() && require_space_after_number_) {
|
463
|
+
AddError("Need space between number and identifier.");
|
464
|
+
} else if (current_char_ == '.') {
|
465
|
+
if (is_float) {
|
466
|
+
AddError(
|
467
|
+
"Already saw decimal point or exponent; can't have another one.");
|
468
|
+
} else {
|
469
|
+
AddError("Hex and octal numbers must be integers.");
|
470
|
+
}
|
471
|
+
}
|
472
|
+
|
473
|
+
return is_float ? TYPE_FLOAT : TYPE_INTEGER;
|
474
|
+
}
|
475
|
+
|
476
|
+
void Tokenizer::ConsumeLineComment(string* content) {
|
477
|
+
if (content != NULL) RecordTo(content);
|
478
|
+
|
479
|
+
while (current_char_ != '\0' && current_char_ != '\n') {
|
480
|
+
NextChar();
|
481
|
+
}
|
482
|
+
TryConsume('\n');
|
483
|
+
|
484
|
+
if (content != NULL) StopRecording();
|
485
|
+
}
|
486
|
+
|
487
|
+
void Tokenizer::ConsumeBlockComment(string* content) {
|
488
|
+
int start_line = line_;
|
489
|
+
int start_column = column_ - 2;
|
490
|
+
|
491
|
+
if (content != NULL) RecordTo(content);
|
492
|
+
|
493
|
+
while (true) {
|
494
|
+
while (current_char_ != '\0' &&
|
495
|
+
current_char_ != '*' &&
|
496
|
+
current_char_ != '/' &&
|
497
|
+
current_char_ != '\n') {
|
498
|
+
NextChar();
|
499
|
+
}
|
500
|
+
|
501
|
+
if (TryConsume('\n')) {
|
502
|
+
if (content != NULL) StopRecording();
|
503
|
+
|
504
|
+
// Consume leading whitespace and asterisk;
|
505
|
+
ConsumeZeroOrMore<WhitespaceNoNewline>();
|
506
|
+
if (TryConsume('*')) {
|
507
|
+
if (TryConsume('/')) {
|
508
|
+
// End of comment.
|
509
|
+
break;
|
510
|
+
}
|
511
|
+
}
|
512
|
+
|
513
|
+
if (content != NULL) RecordTo(content);
|
514
|
+
} else if (TryConsume('*') && TryConsume('/')) {
|
515
|
+
// End of comment.
|
516
|
+
if (content != NULL) {
|
517
|
+
StopRecording();
|
518
|
+
// Strip trailing "*/".
|
519
|
+
content->erase(content->size() - 2);
|
520
|
+
}
|
521
|
+
break;
|
522
|
+
} else if (TryConsume('/') && current_char_ == '*') {
|
523
|
+
// Note: We didn't consume the '*' because if there is a '/' after it
|
524
|
+
// we want to interpret that as the end of the comment.
|
525
|
+
AddError(
|
526
|
+
"\"/*\" inside block comment. Block comments cannot be nested.");
|
527
|
+
} else if (current_char_ == '\0') {
|
528
|
+
AddError("End-of-file inside block comment.");
|
529
|
+
error_collector_->AddError(
|
530
|
+
start_line, start_column, " Comment started here.");
|
531
|
+
if (content != NULL) StopRecording();
|
532
|
+
break;
|
533
|
+
}
|
534
|
+
}
|
535
|
+
}
|
536
|
+
|
537
|
+
Tokenizer::NextCommentStatus Tokenizer::TryConsumeCommentStart() {
|
538
|
+
if (comment_style_ == CPP_COMMENT_STYLE && TryConsume('/')) {
|
539
|
+
if (TryConsume('/')) {
|
540
|
+
return LINE_COMMENT;
|
541
|
+
} else if (TryConsume('*')) {
|
542
|
+
return BLOCK_COMMENT;
|
543
|
+
} else {
|
544
|
+
// Oops, it was just a slash. Return it.
|
545
|
+
current_.type = TYPE_SYMBOL;
|
546
|
+
current_.text = "/";
|
547
|
+
current_.line = line_;
|
548
|
+
current_.column = column_ - 1;
|
549
|
+
current_.end_column = column_;
|
550
|
+
return SLASH_NOT_COMMENT;
|
551
|
+
}
|
552
|
+
} else if (comment_style_ == SH_COMMENT_STYLE && TryConsume('#')) {
|
553
|
+
return LINE_COMMENT;
|
554
|
+
} else {
|
555
|
+
return NO_COMMENT;
|
556
|
+
}
|
557
|
+
}
|
558
|
+
|
559
|
+
// -------------------------------------------------------------------
|
560
|
+
|
561
|
+
bool Tokenizer::Next() {
|
562
|
+
previous_ = current_;
|
563
|
+
|
564
|
+
while (!read_error_) {
|
565
|
+
ConsumeZeroOrMore<Whitespace>();
|
566
|
+
|
567
|
+
switch (TryConsumeCommentStart()) {
|
568
|
+
case LINE_COMMENT:
|
569
|
+
ConsumeLineComment(NULL);
|
570
|
+
continue;
|
571
|
+
case BLOCK_COMMENT:
|
572
|
+
ConsumeBlockComment(NULL);
|
573
|
+
continue;
|
574
|
+
case SLASH_NOT_COMMENT:
|
575
|
+
return true;
|
576
|
+
case NO_COMMENT:
|
577
|
+
break;
|
578
|
+
}
|
579
|
+
|
580
|
+
// Check for EOF before continuing.
|
581
|
+
if (read_error_) break;
|
582
|
+
|
583
|
+
if (LookingAt<Unprintable>() || current_char_ == '\0') {
|
584
|
+
AddError("Invalid control characters encountered in text.");
|
585
|
+
NextChar();
|
586
|
+
// Skip more unprintable characters, too. But, remember that '\0' is
|
587
|
+
// also what current_char_ is set to after EOF / read error. We have
|
588
|
+
// to be careful not to go into an infinite loop of trying to consume
|
589
|
+
// it, so make sure to check read_error_ explicitly before consuming
|
590
|
+
// '\0'.
|
591
|
+
while (TryConsumeOne<Unprintable>() ||
|
592
|
+
(!read_error_ && TryConsume('\0'))) {
|
593
|
+
// Ignore.
|
594
|
+
}
|
595
|
+
|
596
|
+
} else {
|
597
|
+
// Reading some sort of token.
|
598
|
+
StartToken();
|
599
|
+
|
600
|
+
if (TryConsumeOne<Letter>()) {
|
601
|
+
ConsumeZeroOrMore<Alphanumeric>();
|
602
|
+
current_.type = TYPE_IDENTIFIER;
|
603
|
+
} else if (TryConsume('0')) {
|
604
|
+
current_.type = ConsumeNumber(true, false);
|
605
|
+
} else if (TryConsume('.')) {
|
606
|
+
// This could be the beginning of a floating-point number, or it could
|
607
|
+
// just be a '.' symbol.
|
608
|
+
|
609
|
+
if (TryConsumeOne<Digit>()) {
|
610
|
+
// It's a floating-point number.
|
611
|
+
if (previous_.type == TYPE_IDENTIFIER &&
|
612
|
+
current_.line == previous_.line &&
|
613
|
+
current_.column == previous_.end_column) {
|
614
|
+
// We don't accept syntax like "blah.123".
|
615
|
+
error_collector_->AddError(line_, column_ - 2,
|
616
|
+
"Need space between identifier and decimal point.");
|
617
|
+
}
|
618
|
+
current_.type = ConsumeNumber(false, true);
|
619
|
+
} else {
|
620
|
+
current_.type = TYPE_SYMBOL;
|
621
|
+
}
|
622
|
+
} else if (TryConsumeOne<Digit>()) {
|
623
|
+
current_.type = ConsumeNumber(false, false);
|
624
|
+
} else if (TryConsume('\"')) {
|
625
|
+
ConsumeString('\"');
|
626
|
+
current_.type = TYPE_STRING;
|
627
|
+
} else if (TryConsume('\'')) {
|
628
|
+
ConsumeString('\'');
|
629
|
+
current_.type = TYPE_STRING;
|
630
|
+
} else {
|
631
|
+
// Check if the high order bit is set.
|
632
|
+
if (current_char_ & 0x80) {
|
633
|
+
error_collector_->AddError(line_, column_,
|
634
|
+
StringPrintf("Interpreting non ascii codepoint %d.",
|
635
|
+
static_cast<unsigned char>(current_char_)));
|
636
|
+
}
|
637
|
+
NextChar();
|
638
|
+
current_.type = TYPE_SYMBOL;
|
639
|
+
}
|
640
|
+
|
641
|
+
EndToken();
|
642
|
+
return true;
|
643
|
+
}
|
644
|
+
}
|
645
|
+
|
646
|
+
// EOF
|
647
|
+
current_.type = TYPE_END;
|
648
|
+
current_.text.clear();
|
649
|
+
current_.line = line_;
|
650
|
+
current_.column = column_;
|
651
|
+
current_.end_column = column_;
|
652
|
+
return false;
|
653
|
+
}
|
654
|
+
|
655
|
+
namespace {
|
656
|
+
|
657
|
+
// Helper class for collecting comments and putting them in the right places.
|
658
|
+
//
|
659
|
+
// This basically just buffers the most recent comment until it can be decided
|
660
|
+
// exactly where that comment should be placed. When Flush() is called, the
|
661
|
+
// current comment goes into either prev_trailing_comments or detached_comments.
|
662
|
+
// When the CommentCollector is destroyed, the last buffered comment goes into
|
663
|
+
// next_leading_comments.
|
664
|
+
class CommentCollector {
|
665
|
+
public:
|
666
|
+
CommentCollector(string* prev_trailing_comments,
|
667
|
+
vector<string>* detached_comments,
|
668
|
+
string* next_leading_comments)
|
669
|
+
: prev_trailing_comments_(prev_trailing_comments),
|
670
|
+
detached_comments_(detached_comments),
|
671
|
+
next_leading_comments_(next_leading_comments),
|
672
|
+
has_comment_(false),
|
673
|
+
is_line_comment_(false),
|
674
|
+
can_attach_to_prev_(true) {
|
675
|
+
if (prev_trailing_comments != NULL) prev_trailing_comments->clear();
|
676
|
+
if (detached_comments != NULL) detached_comments->clear();
|
677
|
+
if (next_leading_comments != NULL) next_leading_comments->clear();
|
678
|
+
}
|
679
|
+
|
680
|
+
~CommentCollector() {
|
681
|
+
// Whatever is in the buffer is a leading comment.
|
682
|
+
if (next_leading_comments_ != NULL && has_comment_) {
|
683
|
+
comment_buffer_.swap(*next_leading_comments_);
|
684
|
+
}
|
685
|
+
}
|
686
|
+
|
687
|
+
// About to read a line comment. Get the comment buffer pointer in order to
|
688
|
+
// read into it.
|
689
|
+
string* GetBufferForLineComment() {
|
690
|
+
// We want to combine with previous line comments, but not block comments.
|
691
|
+
if (has_comment_ && !is_line_comment_) {
|
692
|
+
Flush();
|
693
|
+
}
|
694
|
+
has_comment_ = true;
|
695
|
+
is_line_comment_ = true;
|
696
|
+
return &comment_buffer_;
|
697
|
+
}
|
698
|
+
|
699
|
+
// About to read a block comment. Get the comment buffer pointer in order to
|
700
|
+
// read into it.
|
701
|
+
string* GetBufferForBlockComment() {
|
702
|
+
if (has_comment_) {
|
703
|
+
Flush();
|
704
|
+
}
|
705
|
+
has_comment_ = true;
|
706
|
+
is_line_comment_ = false;
|
707
|
+
return &comment_buffer_;
|
708
|
+
}
|
709
|
+
|
710
|
+
void ClearBuffer() {
|
711
|
+
comment_buffer_.clear();
|
712
|
+
has_comment_ = false;
|
713
|
+
}
|
714
|
+
|
715
|
+
// Called once we know that the comment buffer is complete and is *not*
|
716
|
+
// connected to the next token.
|
717
|
+
void Flush() {
|
718
|
+
if (has_comment_) {
|
719
|
+
if (can_attach_to_prev_) {
|
720
|
+
if (prev_trailing_comments_ != NULL) {
|
721
|
+
prev_trailing_comments_->append(comment_buffer_);
|
722
|
+
}
|
723
|
+
can_attach_to_prev_ = false;
|
724
|
+
} else {
|
725
|
+
if (detached_comments_ != NULL) {
|
726
|
+
detached_comments_->push_back(comment_buffer_);
|
727
|
+
}
|
728
|
+
}
|
729
|
+
ClearBuffer();
|
730
|
+
}
|
731
|
+
}
|
732
|
+
|
733
|
+
void DetachFromPrev() {
|
734
|
+
can_attach_to_prev_ = false;
|
735
|
+
}
|
736
|
+
|
737
|
+
private:
|
738
|
+
string* prev_trailing_comments_;
|
739
|
+
vector<string>* detached_comments_;
|
740
|
+
string* next_leading_comments_;
|
741
|
+
|
742
|
+
string comment_buffer_;
|
743
|
+
|
744
|
+
// True if any comments were read into comment_buffer_. This can be true even
|
745
|
+
// if comment_buffer_ is empty, namely if the comment was "/**/".
|
746
|
+
bool has_comment_;
|
747
|
+
|
748
|
+
// Is the comment in the comment buffer a line comment?
|
749
|
+
bool is_line_comment_;
|
750
|
+
|
751
|
+
// Is it still possible that we could be reading a comment attached to the
|
752
|
+
// previous token?
|
753
|
+
bool can_attach_to_prev_;
|
754
|
+
};
|
755
|
+
|
756
|
+
} // namespace
|
757
|
+
|
758
|
+
bool Tokenizer::NextWithComments(string* prev_trailing_comments,
|
759
|
+
vector<string>* detached_comments,
|
760
|
+
string* next_leading_comments) {
|
761
|
+
CommentCollector collector(prev_trailing_comments, detached_comments,
|
762
|
+
next_leading_comments);
|
763
|
+
|
764
|
+
if (current_.type == TYPE_START) {
|
765
|
+
collector.DetachFromPrev();
|
766
|
+
} else {
|
767
|
+
// A comment appearing on the same line must be attached to the previous
|
768
|
+
// declaration.
|
769
|
+
ConsumeZeroOrMore<WhitespaceNoNewline>();
|
770
|
+
switch (TryConsumeCommentStart()) {
|
771
|
+
case LINE_COMMENT:
|
772
|
+
ConsumeLineComment(collector.GetBufferForLineComment());
|
773
|
+
|
774
|
+
// Don't allow comments on subsequent lines to be attached to a trailing
|
775
|
+
// comment.
|
776
|
+
collector.Flush();
|
777
|
+
break;
|
778
|
+
case BLOCK_COMMENT:
|
779
|
+
ConsumeBlockComment(collector.GetBufferForBlockComment());
|
780
|
+
|
781
|
+
ConsumeZeroOrMore<WhitespaceNoNewline>();
|
782
|
+
if (!TryConsume('\n')) {
|
783
|
+
// Oops, the next token is on the same line. If we recorded a comment
|
784
|
+
// we really have no idea which token it should be attached to.
|
785
|
+
collector.ClearBuffer();
|
786
|
+
return Next();
|
787
|
+
}
|
788
|
+
|
789
|
+
// Don't allow comments on subsequent lines to be attached to a trailing
|
790
|
+
// comment.
|
791
|
+
collector.Flush();
|
792
|
+
break;
|
793
|
+
case SLASH_NOT_COMMENT:
|
794
|
+
return true;
|
795
|
+
case NO_COMMENT:
|
796
|
+
if (!TryConsume('\n')) {
|
797
|
+
// The next token is on the same line. There are no comments.
|
798
|
+
return Next();
|
799
|
+
}
|
800
|
+
break;
|
801
|
+
}
|
802
|
+
}
|
803
|
+
|
804
|
+
// OK, we are now on the line *after* the previous token.
|
805
|
+
while (true) {
|
806
|
+
ConsumeZeroOrMore<WhitespaceNoNewline>();
|
807
|
+
|
808
|
+
switch (TryConsumeCommentStart()) {
|
809
|
+
case LINE_COMMENT:
|
810
|
+
ConsumeLineComment(collector.GetBufferForLineComment());
|
811
|
+
break;
|
812
|
+
case BLOCK_COMMENT:
|
813
|
+
ConsumeBlockComment(collector.GetBufferForBlockComment());
|
814
|
+
|
815
|
+
// Consume the rest of the line so that we don't interpret it as a
|
816
|
+
// blank line the next time around the loop.
|
817
|
+
ConsumeZeroOrMore<WhitespaceNoNewline>();
|
818
|
+
TryConsume('\n');
|
819
|
+
break;
|
820
|
+
case SLASH_NOT_COMMENT:
|
821
|
+
return true;
|
822
|
+
case NO_COMMENT:
|
823
|
+
if (TryConsume('\n')) {
|
824
|
+
// Completely blank line.
|
825
|
+
collector.Flush();
|
826
|
+
collector.DetachFromPrev();
|
827
|
+
} else {
|
828
|
+
bool result = Next();
|
829
|
+
if (!result ||
|
830
|
+
current_.text == "}" ||
|
831
|
+
current_.text == "]" ||
|
832
|
+
current_.text == ")") {
|
833
|
+
// It looks like we're at the end of a scope. In this case it
|
834
|
+
// makes no sense to attach a comment to the following token.
|
835
|
+
collector.Flush();
|
836
|
+
}
|
837
|
+
return result;
|
838
|
+
}
|
839
|
+
break;
|
840
|
+
}
|
841
|
+
}
|
842
|
+
}
|
843
|
+
|
844
|
+
// -------------------------------------------------------------------
|
845
|
+
// Token-parsing helpers. Remember that these don't need to report
|
846
|
+
// errors since any errors should already have been reported while
|
847
|
+
// tokenizing. Also, these can assume that whatever text they
|
848
|
+
// are given is text that the tokenizer actually parsed as a token
|
849
|
+
// of the given type.
|
850
|
+
|
851
|
+
bool Tokenizer::ParseInteger(const string& text, uint64 max_value,
|
852
|
+
uint64* output) {
|
853
|
+
// Sadly, we can't just use strtoul() since it is only 32-bit and strtoull()
|
854
|
+
// is non-standard. I hate the C standard library. :(
|
855
|
+
|
856
|
+
// return strtoull(text.c_str(), NULL, 0);
|
857
|
+
|
858
|
+
const char* ptr = text.c_str();
|
859
|
+
int base = 10;
|
860
|
+
if (ptr[0] == '0') {
|
861
|
+
if (ptr[1] == 'x' || ptr[1] == 'X') {
|
862
|
+
// This is hex.
|
863
|
+
base = 16;
|
864
|
+
ptr += 2;
|
865
|
+
} else {
|
866
|
+
// This is octal.
|
867
|
+
base = 8;
|
868
|
+
}
|
869
|
+
}
|
870
|
+
|
871
|
+
uint64 result = 0;
|
872
|
+
for (; *ptr != '\0'; ptr++) {
|
873
|
+
int digit = DigitValue(*ptr);
|
874
|
+
GOOGLE_LOG_IF(DFATAL, digit < 0 || digit >= base)
|
875
|
+
<< " Tokenizer::ParseInteger() passed text that could not have been"
|
876
|
+
" tokenized as an integer: " << CEscape(text);
|
877
|
+
if (digit > max_value || result > (max_value - digit) / base) {
|
878
|
+
// Overflow.
|
879
|
+
return false;
|
880
|
+
}
|
881
|
+
result = result * base + digit;
|
882
|
+
}
|
883
|
+
|
884
|
+
*output = result;
|
885
|
+
return true;
|
886
|
+
}
|
887
|
+
|
888
|
+
double Tokenizer::ParseFloat(const string& text) {
|
889
|
+
const char* start = text.c_str();
|
890
|
+
char* end;
|
891
|
+
double result = NoLocaleStrtod(start, &end);
|
892
|
+
|
893
|
+
// "1e" is not a valid float, but if the tokenizer reads it, it will
|
894
|
+
// report an error but still return it as a valid token. We need to
|
895
|
+
// accept anything the tokenizer could possibly return, error or not.
|
896
|
+
if (*end == 'e' || *end == 'E') {
|
897
|
+
++end;
|
898
|
+
if (*end == '-' || *end == '+') ++end;
|
899
|
+
}
|
900
|
+
|
901
|
+
// If the Tokenizer had allow_f_after_float_ enabled, the float may be
|
902
|
+
// suffixed with the letter 'f'.
|
903
|
+
if (*end == 'f' || *end == 'F') {
|
904
|
+
++end;
|
905
|
+
}
|
906
|
+
|
907
|
+
GOOGLE_LOG_IF(DFATAL, end - start != text.size() || *start == '-')
|
908
|
+
<< " Tokenizer::ParseFloat() passed text that could not have been"
|
909
|
+
" tokenized as a float: " << CEscape(text);
|
910
|
+
return result;
|
911
|
+
}
|
912
|
+
|
913
|
+
// Helper to append a Unicode code point to a string as UTF8, without bringing
|
914
|
+
// in any external dependencies.
|
915
|
+
static void AppendUTF8(uint32 code_point, string* output) {
|
916
|
+
uint32 tmp = 0;
|
917
|
+
int len = 0;
|
918
|
+
if (code_point <= 0x7f) {
|
919
|
+
tmp = code_point;
|
920
|
+
len = 1;
|
921
|
+
} else if (code_point <= 0x07ff) {
|
922
|
+
tmp = 0x0000c080 |
|
923
|
+
((code_point & 0x07c0) << 2) |
|
924
|
+
(code_point & 0x003f);
|
925
|
+
len = 2;
|
926
|
+
} else if (code_point <= 0xffff) {
|
927
|
+
tmp = 0x00e08080 |
|
928
|
+
((code_point & 0xf000) << 4) |
|
929
|
+
((code_point & 0x0fc0) << 2) |
|
930
|
+
(code_point & 0x003f);
|
931
|
+
len = 3;
|
932
|
+
} else if (code_point <= 0x1fffff) {
|
933
|
+
tmp = 0xf0808080 |
|
934
|
+
((code_point & 0x1c0000) << 6) |
|
935
|
+
((code_point & 0x03f000) << 4) |
|
936
|
+
((code_point & 0x000fc0) << 2) |
|
937
|
+
(code_point & 0x003f);
|
938
|
+
len = 4;
|
939
|
+
} else {
|
940
|
+
// UTF-16 is only defined for code points up to 0x10FFFF, and UTF-8 is
|
941
|
+
// normally only defined up to there as well.
|
942
|
+
StringAppendF(output, "\\U%08x", code_point);
|
943
|
+
return;
|
944
|
+
}
|
945
|
+
tmp = ghtonl(tmp);
|
946
|
+
output->append(reinterpret_cast<const char*>(&tmp) + sizeof(tmp) - len, len);
|
947
|
+
}
|
948
|
+
|
949
|
+
// Try to read <len> hex digits from ptr, and stuff the numeric result into
|
950
|
+
// *result. Returns true if that many digits were successfully consumed.
|
951
|
+
static bool ReadHexDigits(const char* ptr, int len, uint32* result) {
|
952
|
+
*result = 0;
|
953
|
+
if (len == 0) return false;
|
954
|
+
for (const char* end = ptr + len; ptr < end; ++ptr) {
|
955
|
+
if (*ptr == '\0') return false;
|
956
|
+
*result = (*result << 4) + DigitValue(*ptr);
|
957
|
+
}
|
958
|
+
return true;
|
959
|
+
}
|
960
|
+
|
961
|
+
// Handling UTF-16 surrogate pairs. UTF-16 encodes code points in the range
|
962
|
+
// 0x10000...0x10ffff as a pair of numbers, a head surrogate followed by a trail
|
963
|
+
// surrogate. These numbers are in a reserved range of Unicode code points, so
|
964
|
+
// if we encounter such a pair we know how to parse it and convert it into a
|
965
|
+
// single code point.
|
966
|
+
static const uint32 kMinHeadSurrogate = 0xd800;
|
967
|
+
static const uint32 kMaxHeadSurrogate = 0xdc00;
|
968
|
+
static const uint32 kMinTrailSurrogate = 0xdc00;
|
969
|
+
static const uint32 kMaxTrailSurrogate = 0xe000;
|
970
|
+
|
971
|
+
static inline bool IsHeadSurrogate(uint32 code_point) {
|
972
|
+
return (code_point >= kMinHeadSurrogate) && (code_point < kMaxHeadSurrogate);
|
973
|
+
}
|
974
|
+
|
975
|
+
static inline bool IsTrailSurrogate(uint32 code_point) {
|
976
|
+
return (code_point >= kMinTrailSurrogate) &&
|
977
|
+
(code_point < kMaxTrailSurrogate);
|
978
|
+
}
|
979
|
+
|
980
|
+
// Combine a head and trail surrogate into a single Unicode code point.
|
981
|
+
static uint32 AssembleUTF16(uint32 head_surrogate, uint32 trail_surrogate) {
|
982
|
+
GOOGLE_DCHECK(IsHeadSurrogate(head_surrogate));
|
983
|
+
GOOGLE_DCHECK(IsTrailSurrogate(trail_surrogate));
|
984
|
+
return 0x10000 + (((head_surrogate - kMinHeadSurrogate) << 10) |
|
985
|
+
(trail_surrogate - kMinTrailSurrogate));
|
986
|
+
}
|
987
|
+
|
988
|
+
// Convert the escape sequence parameter to a number of expected hex digits.
|
989
|
+
static inline int UnicodeLength(char key) {
|
990
|
+
if (key == 'u') return 4;
|
991
|
+
if (key == 'U') return 8;
|
992
|
+
return 0;
|
993
|
+
}
|
994
|
+
|
995
|
+
// Given a pointer to the 'u' or 'U' starting a Unicode escape sequence, attempt
|
996
|
+
// to parse that sequence. On success, returns a pointer to the first char
|
997
|
+
// beyond that sequence, and fills in *code_point. On failure, returns ptr
|
998
|
+
// itself.
|
999
|
+
static const char* FetchUnicodePoint(const char* ptr, uint32* code_point) {
|
1000
|
+
const char* p = ptr;
|
1001
|
+
// Fetch the code point.
|
1002
|
+
const int len = UnicodeLength(*p++);
|
1003
|
+
if (!ReadHexDigits(p, len, code_point))
|
1004
|
+
return ptr;
|
1005
|
+
p += len;
|
1006
|
+
|
1007
|
+
// Check if the code point we read is a "head surrogate." If so, then we
|
1008
|
+
// expect it to be immediately followed by another code point which is a valid
|
1009
|
+
// "trail surrogate," and together they form a UTF-16 pair which decodes into
|
1010
|
+
// a single Unicode point. Trail surrogates may only use \u, not \U.
|
1011
|
+
if (IsHeadSurrogate(*code_point) && *p == '\\' && *(p + 1) == 'u') {
|
1012
|
+
uint32 trail_surrogate;
|
1013
|
+
if (ReadHexDigits(p + 2, 4, &trail_surrogate) &&
|
1014
|
+
IsTrailSurrogate(trail_surrogate)) {
|
1015
|
+
*code_point = AssembleUTF16(*code_point, trail_surrogate);
|
1016
|
+
p += 6;
|
1017
|
+
}
|
1018
|
+
// If this failed, then we just emit the head surrogate as a code point.
|
1019
|
+
// It's bogus, but so is the string.
|
1020
|
+
}
|
1021
|
+
|
1022
|
+
return p;
|
1023
|
+
}
|
1024
|
+
|
1025
|
+
// The text string must begin and end with single or double quote
|
1026
|
+
// characters.
|
1027
|
+
void Tokenizer::ParseStringAppend(const string& text, string* output) {
|
1028
|
+
// Reminder: text[0] is always a quote character. (If text is
|
1029
|
+
// empty, it's invalid, so we'll just return).
|
1030
|
+
const size_t text_size = text.size();
|
1031
|
+
if (text_size == 0) {
|
1032
|
+
GOOGLE_LOG(DFATAL)
|
1033
|
+
<< " Tokenizer::ParseStringAppend() passed text that could not"
|
1034
|
+
" have been tokenized as a string: " << CEscape(text);
|
1035
|
+
return;
|
1036
|
+
}
|
1037
|
+
|
1038
|
+
// Reserve room for new string. The branch is necessary because if
|
1039
|
+
// there is already space available the reserve() call might
|
1040
|
+
// downsize the output.
|
1041
|
+
const size_t new_len = text_size + output->size();
|
1042
|
+
if (new_len > output->capacity()) {
|
1043
|
+
output->reserve(new_len);
|
1044
|
+
}
|
1045
|
+
|
1046
|
+
// Loop through the string copying characters to "output" and
|
1047
|
+
// interpreting escape sequences. Note that any invalid escape
|
1048
|
+
// sequences or other errors were already reported while tokenizing.
|
1049
|
+
// In this case we do not need to produce valid results.
|
1050
|
+
for (const char* ptr = text.c_str() + 1; *ptr != '\0'; ptr++) {
|
1051
|
+
if (*ptr == '\\' && ptr[1] != '\0') {
|
1052
|
+
// An escape sequence.
|
1053
|
+
++ptr;
|
1054
|
+
|
1055
|
+
if (OctalDigit::InClass(*ptr)) {
|
1056
|
+
// An octal escape. May one, two, or three digits.
|
1057
|
+
int code = DigitValue(*ptr);
|
1058
|
+
if (OctalDigit::InClass(ptr[1])) {
|
1059
|
+
++ptr;
|
1060
|
+
code = code * 8 + DigitValue(*ptr);
|
1061
|
+
}
|
1062
|
+
if (OctalDigit::InClass(ptr[1])) {
|
1063
|
+
++ptr;
|
1064
|
+
code = code * 8 + DigitValue(*ptr);
|
1065
|
+
}
|
1066
|
+
output->push_back(static_cast<char>(code));
|
1067
|
+
|
1068
|
+
} else if (*ptr == 'x') {
|
1069
|
+
// A hex escape. May zero, one, or two digits. (The zero case
|
1070
|
+
// will have been caught as an error earlier.)
|
1071
|
+
int code = 0;
|
1072
|
+
if (HexDigit::InClass(ptr[1])) {
|
1073
|
+
++ptr;
|
1074
|
+
code = DigitValue(*ptr);
|
1075
|
+
}
|
1076
|
+
if (HexDigit::InClass(ptr[1])) {
|
1077
|
+
++ptr;
|
1078
|
+
code = code * 16 + DigitValue(*ptr);
|
1079
|
+
}
|
1080
|
+
output->push_back(static_cast<char>(code));
|
1081
|
+
|
1082
|
+
} else if (*ptr == 'u' || *ptr == 'U') {
|
1083
|
+
uint32 unicode;
|
1084
|
+
const char* end = FetchUnicodePoint(ptr, &unicode);
|
1085
|
+
if (end == ptr) {
|
1086
|
+
// Failure: Just dump out what we saw, don't try to parse it.
|
1087
|
+
output->push_back(*ptr);
|
1088
|
+
} else {
|
1089
|
+
AppendUTF8(unicode, output);
|
1090
|
+
ptr = end - 1; // Because we're about to ++ptr.
|
1091
|
+
}
|
1092
|
+
} else {
|
1093
|
+
// Some other escape code.
|
1094
|
+
output->push_back(TranslateEscape(*ptr));
|
1095
|
+
}
|
1096
|
+
|
1097
|
+
} else if (*ptr == text[0] && ptr[1] == '\0') {
|
1098
|
+
// Ignore final quote matching the starting quote.
|
1099
|
+
} else {
|
1100
|
+
output->push_back(*ptr);
|
1101
|
+
}
|
1102
|
+
}
|
1103
|
+
}
|
1104
|
+
|
1105
|
+
template<typename CharacterClass>
|
1106
|
+
static bool AllInClass(const string& s) {
|
1107
|
+
for (int i = 0; i < s.size(); ++i) {
|
1108
|
+
if (!CharacterClass::InClass(s[i]))
|
1109
|
+
return false;
|
1110
|
+
}
|
1111
|
+
return true;
|
1112
|
+
}
|
1113
|
+
|
1114
|
+
bool Tokenizer::IsIdentifier(const string& text) {
|
1115
|
+
// Mirrors IDENTIFIER definition in Tokenizer::Next() above.
|
1116
|
+
if (text.size() == 0)
|
1117
|
+
return false;
|
1118
|
+
if (!Letter::InClass(text.at(0)))
|
1119
|
+
return false;
|
1120
|
+
if (!AllInClass<Alphanumeric>(text.substr(1)))
|
1121
|
+
return false;
|
1122
|
+
return true;
|
1123
|
+
}
|
1124
|
+
|
1125
|
+
} // namespace io
|
1126
|
+
} // namespace protobuf
|
1127
|
+
} // namespace google
|